Free Motion Capture for Everyone πβ¨
TypeScript
refactor to run everything through `CameraManager`
Jone7bbb4d5fad0
diff --git a/src/api/routes/camera/cam_data_ws.py b/src/api/routes/camera/cam_data_ws.pyindex f2eb25ea..f19d1643 100644--- a/src/api/routes/camera/cam_data_ws.py+++ b/src/api/routes/camera/cam_data_ws.py@@ -6,7 +6,7 @@ from fastapi import APIRouter, WebSocketfrom src.api.services.board_detect_service import BoardDetectServicefrom src.api.services.mediapipe_detect_service import MediapipeSkeletonDetectionService-from src.cameras.multicam_manager.cv_camera_manager import CVCameraManager+from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManagerlogger = logging.getLogger(__name__)@@ -50,7 +50,7 @@ async def skeleton_detection_as_ws(await web_socket.send_bytes(frame.tobytes())try:- await MediapipeSkeletonDetectionService(CVCameraManager()).run_as_loop(+ await MediapipeSkeletonDetectionService(OpenCVCameraManager()).run_as_loop(webcam_id=webcam_id, cb=websocket_send, model_complexity=model_complexity)except:@@ -64,5 +64,5 @@ async def begin_mediapipe_skeleton_detection(model_complexity: int):"""model_complexity can be 1 (faster, less accurate) or 2 (slower, more accurate)"""- service = MediapipeSkeletonDetectionService(CVCameraManager())+ service = MediapipeSkeletonDetectionService(OpenCVCameraManager())service.run(model_complexity)diff --git a/src/api/services/board_detect_service.py b/src/api/services/board_detect_service.pyindex 993e7f74..1887791a 100644--- a/src/api/services/board_detect_service.py+++ b/src/api/services/board_detect_service.py@@ -1,7 +1,7 @@from typing import NamedTuplefrom src.cameras.multicam_manager import CVCameraManager-from src.core_processor.board_detection.board_detection import BoardDetection+from src.core_processor.board_detection.board_detection import BoardDetectorclass ImageResponse(NamedTuple):@@ -14,7 +14,7 @@ class BoardDetectService:self._cam_manager = cam_managerasync def run(self):- return BoardDetection(self._cam_manager).process()+ return BoardDetector(self._cam_manager).process()async def run_detection_on_cam_id(self, webcam_id: str, cb=None):- await BoardDetection(self._cam_manager).process_by_cam_id(webcam_id, cb)+ await BoardDetector(self._cam_manager).process_by_cam_id(webcam_id, cb)diff --git a/src/api/services/mediapipe_detect_service.py b/src/api/services/mediapipe_detect_service.pyindex 87b4b234..e03c85a6 100644--- a/src/api/services/mediapipe_detect_service.py+++ b/src/api/services/mediapipe_detect_service.py@@ -1,11 +1,11 @@-from src.cameras.multicam_manager.cv_camera_manager import CVCameraManager+from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManagerfrom src.core_processor.mediapipe_skeleton_detection.mediapipe_skeleton_detection import (MediapipeSkeletonDetection,)class MediapipeSkeletonDetectionService:- def __init__(self, cam_manager: CVCameraManager):+ def __init__(self, cam_manager: OpenCVCameraManager):self._cam_manager = cam_managerdef run(self, model_complexity: int):diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.pyindex 51c4cf8f..ecceb53f 100644--- a/src/cameras/capture/opencv_camera/opencv_camera.py+++ b/src/cameras/capture/opencv_camera/opencv_camera.py@@ -9,6 +9,7 @@ from pydantic import BaseModelfrom src.cameras.capture.frame_payload import FramePayloadfrom src.cameras.capture.opencv_camera.frame_grabber import FrameThread+from src.core_processor.camera_calibration.camera_calibrator import CameraCalibrationDatalogger = logging.getLogger(__name__)@@ -38,6 +39,22 @@ class OpenCVCamera:self._name = f"Camera {self._config.webcam_id}"self._opencv_video_capture_object: cv2.VideoCapture = Noneself._running_thread: FrameThread = None+ self._camera_calibration_data = None++ @property+ def camera_calibration_data(self):+ if self._camera_calibration_data is not None:+ return self._camera_calibration_data++ if self._camera_calibration_data is None and self.image_width is not None:+ return self.camera_calibration_data(CameraCalibrationData(self.image_width, self.image_height))++ logger.warning('Can\'t request a cameras calibration info until the video capture is initialized and the image_width and image_height are known')+ return None++ @camera_calibration_data.setter+ def camera_calibration_data(self, camera_calibration_data: CameraCalibrationData):+ self._camera_calibration_data = camera_calibration_data@propertydef webcam_id_as_str(self):@@ -89,6 +106,7 @@ class OpenCVCamera:logger.debug(f"Camera found at port number {self._config.webcam_id}")fps_input_stream = int(self._opencv_video_capture_object.get(5))logger.debug("FPS of webcam hardware/input stream: {}".format(fps_input_stream))+return successdef start_frame_capture(self):@@ -108,15 +126,23 @@ class OpenCVCamera:webcam_id=self.webcam_id_as_str,get_next_frame=self.get_next_frame,save_video=self._config.save_video,- frame_width=self.get_frame_width(),- frame_height=self.get_frame_height(),+ frame_width=self.image_width,+ frame_height=self.image_height,)- def get_frame_width(self):- return int(self._opencv_video_capture_object.get(3))+ @property+ def image_width(self):+ try:+ return int(self._opencv_video_capture_object.get(3))+ except:+ return None- def get_frame_height(self):- return int(self._opencv_video_capture_object.get(4))+ @property+ def image_height(self):+ try:+ return int(self._opencv_video_capture_object.get(4))+ except:+ return Nonedef _apply_configuration(self):# set camera stream parametersdiff --git a/src/cameras/detection/cam_detection.py b/src/cameras/detection/cam_detection.pyindex d51fdad0..aaaab6c7 100644--- a/src/cameras/detection/cam_detection.py+++ b/src/cameras/detection/cam_detection.py@@ -3,7 +3,7 @@ import platformimport cv2-from src.cameras.detection.models import FoundCamerasResponse, RawCamera+from src.cameras.detection.models import FoundCamerasResponseCAM_CHECK_NUM = 20@@ -15,27 +15,24 @@ class DetectPossibleCameras:cv2_backend = self._determine_backend()cams_to_use_list = []- for cam_id in range(CAM_CHECK_NUM):- cap = cv2.VideoCapture(cam_id, cv2_backend)+ for this_usb_port_number in range(CAM_CHECK_NUM):+ cap = cv2.VideoCapture(this_usb_port_number, cv2_backend)success, image = cap.read()if success:try:- cams_to_use_list.append(- RawCamera(- webcam_id=cam_id,- )- )+ cams_to_use_list.append(this_usb_port_number)finally:cap.release()return FoundCamerasResponse(- camera_found_count=len(cams_to_use_list),- cams_to_use=cams_to_use_list,+ number_of_cameras_found=len(cams_to_use_list),+ list_of_usb_port_numbers_with_cameras_attached=cams_to_use_list,cv2_backend=cv2_backend,)- def _determine_backend(self):+ @staticmethod+ def _determine_backend():if platform.system() == "Windows":return cv2.CAP_DSHOWelse:diff --git a/src/cameras/detection/models.py b/src/cameras/detection/models.pyindex dee5c0f8..d96b3a60 100644--- a/src/cameras/detection/models.py+++ b/src/cameras/detection/models.py@@ -3,11 +3,7 @@ from typing import Listfrom pydantic import BaseModel-class RawCamera(BaseModel):- webcam_id: str--class FoundCamerasResponse(BaseModel):- camera_found_count: int- cams_to_use: List[RawCamera]+ number_of_cameras_found: int+ list_of_usb_port_numbers_with_cameras_attached: List[int]cv2_backend: intdiff --git a/src/cameras/multicam_manager/cv_camera_manager.py b/src/cameras/multicam_manager/cv_camera_manager.pyindex 1ac6a432..42d13006 100644--- a/src/cameras/multicam_manager/cv_camera_manager.py+++ b/src/cameras/multicam_manager/cv_camera_manager.py@@ -9,10 +9,11 @@ from src.api.services.user_config import UserConfigServicefrom src.cameras.detection.cam_singleton import get_or_create_camsfrom src.cameras.capture.opencv_camera.opencv_camera import OpenCVCamerafrom src.cameras.persistence.video_writer.video_writer import VideoWriter+from src.core_processor.camera_calibration.camera_calibrator import CameraCalibrationDatalogger = logging.getLogger(__name__)-_cv_cams = None+_open_cv_cameras = Noneclass CamAndWriterResponse(BaseModel):@@ -23,41 +24,42 @@ class CamAndWriterResponse(BaseModel):arbitrary_types_allowed = True-class CVCameraManager:+class OpenCVCameraManager:def __init__(self):self._config_service = UserConfigService()self._detected_cams_data = get_or_create_cams()- global _cv_cams- if _cv_cams is None:+ global _open_cv_cameras+ if _open_cv_cameras is None:logger.info("Creating cams.")# we create the _cv_cams /once/, and reuse it for the lifetime of the session- _cv_cams = self._create_opencv_cams()- self._cv_cams = _cv_cams+ _open_cv_cameras = self._create_opencv_cameras(calibrate_cameras=True)+ self._open_cv_camera_objects = _open_cv_cameraselse:logger.info("Reusing already created resources cam resources.")- self._cv_cams = _cv_cams+ self._open_cv_camera_objects = _open_cv_cameras@propertydef available_webcam_ids(self):- return [cv_cam.webcam_id_as_str for cv_cam in self._cv_cams]+ return [cv_cam.webcam_id_as_str for cv_cam in self._open_cv_camera_objects]@property- def cv_cams(self):- return self._cv_cams+ def open_cv_cameras(self):+ return self._open_cv_camera_objectsdef cv_cam_by_id(self, webcam_id: str):- for cam in self._cv_cams:+ for cam in self._open_cv_camera_objects:if cam.webcam_id_as_str == str(webcam_id):return camreturn None- def _create_opencv_cams(self):- raw_webcam_obj = self._detected_cams_data.cams_to_use- cv_cams: List[OpenCVCamera] = []- for webcam in raw_webcam_obj:- single_config = self._config_service.webcam_config_by_id(webcam.webcam_id)- cv_cams.append(OpenCVCamera(config=single_config))- return cv_cams+ def _create_opencv_cameras(self, calibrate_cameras=True):+ list_of_usb_port_numbers_with_cameras_attached = self._detected_cams_data.list_of_usb_port_numbers_with_cameras_attached+ open_cv_cameras: List[OpenCVCamera] = []+ for this_port_number in list_of_usb_port_numbers_with_cameras_attached:+ single_camera_config = self._config_service.webcam_config_by_id(this_port_number)+ this_opencv_camera = OpenCVCamera(config=single_camera_config)+ open_cv_cameras.append(this_opencv_camera)+ return open_cv_cameras@contextmanagerdef start_capture_session_single_cam(@@ -94,7 +96,7 @@ class CVCameraManager:def _start_frame_capture_all_cams(self) -> Dict[str, VideoWriter]:d = {}- for cv_cam in self._cv_cams:+ for cv_cam in self._open_cv_camera_objects:cv_cam.connect()cv_cam.start_frame_capture()d[cv_cam.webcam_id_as_str] = VideoWriter()@@ -103,7 +105,7 @@ class CVCameraManager:def _start_frame_capture_on_cam_id(self, webcam_id: str) -> VideoWriter:filtered_cams = list(- filter(lambda c: c.webcam_id_as_str == str(webcam_id), self._cv_cams)+ filter(lambda c: c.webcam_id_as_str == str(webcam_id), self._open_cv_camera_objects))assert (len(filtered_cams) == 1@@ -114,9 +116,9 @@ class CVCameraManager:return VideoWriter()def _stop_frame_capture_all_cams(self):- for cv_cam in self._cv_cams:+ for cv_cam in self._open_cv_camera_objects:cv_cam.stop_frame_capture()def _close_all_cameras(self):- for cv_cam in self._cv_cams:+ for cv_cam in self._open_cv_camera_objects:cv_cam.close()diff --git a/src/core_processor/board_detection/board_detection.py b/src/core_processor/board_detection/board_detection.pyindex 093df174..ad972b87 100644--- a/src/core_processor/board_detection/board_detection.py+++ b/src/core_processor/board_detection/board_detection.py@@ -1,26 +1,16 @@import logging-import time-import tracebackfrom dataclasses import dataclass-from pathlib import Path-from types import FunctionType-from typing import Any, Callable, Optionalimport cv2import numpy as npfrom src.cameras.capture.frame_payload import FramePayload-from src.cameras.multicam_manager.cv_camera_manager import CVCameraManager-from src.cameras.persistence.video_writer.video_writer import SaveOptionsfrom src.core_processor.board_detection.charuco_constants import aruco_marker_dict, charuco_board_object, \number_of_charuco_cornersfrom src.core_processor.board_detection.detect_charuco_board import detect_charuco_board, CharucoViewDatafrom src.core_processor.board_detection.charuco_image_annotator import (annotate_image_with_charuco_data,)-from src.core_processor.fps.fps_counter import FPSCamCounter-from src.core_processor.show_cam_window import show_cam_window-from src.core_processor.utils.image_fps_writer import write_fps_to_imagelogger = logging.getLogger(__name__)@@ -32,9 +22,9 @@ class CharucoFramePayload:charuco_view_data: CharucoViewData-class BoardDetection:+class BoardDetector:@staticmethod- def detect_charuco_board_in_camera_stream(cv_cam)->CharucoFramePayload:+ def detect_charuco_board_in_camera_stream(cv_cam) -> CharucoFramePayload:raw_frame_payload = cv_cam.latest_frameif not raw_frame_payload.success:@@ -59,6 +49,7 @@ class BoardDetection:raw_frame_payload=raw_frame_payload,charuco_view_data=charuco_view_data)+@staticmethoddef detect_charuco_board(image) -> CharucoViewData:"""diff --git a/src/core_processor/camera_calibration/camera_calibrator.py b/src/core_processor/camera_calibration/camera_calibrator.pynew file mode 100644index 00000000..7847db55--- /dev/null+++ b/src/core_processor/camera_calibration/camera_calibrator.py@@ -0,0 +1,37 @@+from dataclasses import dataclass+from typing import List++import numpy as np++from src.cameras.capture.frame_payload import FramePayload+from src.core_processor.camera_calibration.lens_distortion_calibrator import LensDistortionCalibrator, \+ LensDistortionCalibrationData+++@dataclass+class CameraCalibrationData:+ image_width: int+ image_height: int+ lens_distortion_calibration_data: LensDistortionCalibrationData = None+ camera_translation_relative_to_camera0: np.ndarray = np.zeros((3, 1))+ camera_rotation_relative_to_camera0: np.ndarray = np.zeros((3, 1))++ def __init__(self, image_width, image_height):+ self.image_width = image_width+ self.image_height = image_height+ self.lens_distortion_calibration_data = LensDistortionCalibrationData(self.image_width,+ self.image_height)+++class CameraCalibrator:+ def __init__(self,+ lens_distortion_calibrator: LensDistortionCalibrator = LensDistortionCalibrator()+ ):+ self._lens_distortion_calibrator = lens_distortion_calibrator++ def calibrate(self, this_frame: FramePayload):+ self.calibrate_lens_distortion(this_frame)+ # something something cameras 6DoF++ def calibrate_lens_distortion(self, this_frame):+ self._lens_distortion_calibrator.process_incoming_frame(this_frame)diff --git a/src/core_processor/camera_calibration/camera_calibration.py b/src/core_processor/camera_calibration/lens_distortion_calibrator.pysimilarity index 65%rename from src/core_processor/camera_calibration/camera_calibration.pyrename to src/core_processor/camera_calibration/lens_distortion_calibrator.pyindex 8ead5208..dd56253c 100644--- a/src/core_processor/camera_calibration/camera_calibration.py+++ b/src/core_processor/camera_calibration/lens_distortion_calibrator.py@@ -1,49 +1,44 @@-import logging-import tracebackfrom dataclasses import dataclass-import time-from pathlib import Path-from typing import Callableimport cv2-import matplotlib.pyplot as plt-import pyqtgraph as pgimport numpy as npfrom rich import print-from src.cameras.multicam_manager.cv_camera_manager import CVCameraManager-from src.cameras.persistence.video_writer.save_options import SaveOptions-from src.core_processor.board_detection.board_detection import BoardDetection, CharucoFramePayload+from src.core_processor.board_detection.board_detection import BoardDetector, CharucoFramePayloadfrom src.core_processor.board_detection.charuco_constants import charuco_board_objectfrom src.core_processor.board_detection.detect_charuco_board import CharucoViewDatafrom src.core_processor.camera_calibration.calibration_diagnostics_visualizer import CalibrationDiagnosticsVisualizer-from src.core_processor.fps.fps_counter import FPSCamCounter-from src.core_processor.show_cam_window import show_cam_window-logger = logging.getLogger(__name__)@dataclass-class SingleCameraCalibrationData:+class LensDistortionCalibrationData:+ image_width: int+ image_height: intreprojection_error: float = Nonecamera_matrix: np.ndarray = None+ number_of_lens_distortion_coefficients: int = 4lens_distortion_coefficients: np.ndarray = None- image_width: int = None- image_height: int = Nonerotation_vectors_of_the_board: np.ndarray = Nonetranslation_vectors_of_the_board: np.ndarray = Nonelens_distortion_std_dev: float = Nonecamera_location_std_dev: float = None+ def __post_init__(self):+ if self.camera_matrix is None:+ self.camera_matrix = np.array([[float(self.image_width), 0., self.image_width / 2.],+ [0., float(self.image_width), self.image_height / 2.],+ [0., 0., 1.]])-class CameraCalibration:+ if self.lens_distortion_coefficients is None:+ self.lens_distortion_coefficients = np.zeros((self.number_of_lens_distortion_coefficients, 1))+++class LensDistortionCalibrator:def __init__(self,- board_detection: BoardDetection = BoardDetection(),- cam_manager: CVCameraManager = CVCameraManager(),+ board_detection_object: BoardDetector = BoardDetector(),show_calibration_diagnostics_visualizer=True, ):- self._board_detection_object = board_detection- self._cam_manager = cam_manager- self._current_calibration = SingleCameraCalibrationData()- self._start_time = time.time()+ self._board_detection_object = board_detection_object+ self._current_calibration: LensDistortionCalibrationData = Noneself._all_charuco_views = []self._num_charuco_views = 0self._best_combo_of_charuco_views = []@@ -53,112 +48,7 @@ class CameraCalibration:self.calibration_diagnostics_visualizer = CalibrationDiagnosticsVisualizer()- async def process_by_cam_id(self, webcam_id: str, cb):- with self._cam_manager.start_capture_session_single_cam(- webcam_id=webcam_id,- ) as session_obj:- fps_manager = FPSCamCounter(self._cam_manager.available_webcam_ids)- fps_manager.start_all()- cv_cam = session_obj.cv_cam- writer = session_obj.writer- try:- while True:- if not cv_cam.is_capturing_frames:- return- charuco_frame_payload = self._board_detection_object.detect_charuco_board_in_camera_stream(cv_cam)- if cb and charuco_frame_payload.annotated_frame_image is not None:- writer.write(charuco_frame_payload.raw_frame_payload)- await cb(charuco_frame_payload.annotated_frame_image)- except Exception as e:- logger.error("Printing traceback")- traceback.print_exc()- raise e- finally:- writer = session_obj.writer- options = SaveOptions(- writer_dir=Path().joinpath(- cv_cam.session_writer_base_path,- "board_detection",- f"webcam_{cv_cam.webcam_id_as_str}",- ),- fps=fps_manager.current_fps_for(cv_cam.webcam_id_as_str),- frame_width=cv_cam.get_frame_width(),- frame_height=cv_cam.get_frame_height(),- )- writer.save(options)--- def perform_calibration(self):- self.process(- show_window=True,- save_video=False,- post_processed_frame_cb=self._calibration_work,- )-- def process(- self,- post_processed_frame_cb: Callable[[str, CharucoFramePayload], None] = None,- show_window=True,- save_video=True,- ):- """- Opens Cameras using OpenCV and begins image processing for charuco board- If return images is true, the images are returned to the caller- """- # if its already capturing frames, this is a no-op-- with self._cam_manager.start_capture_session_all_cams() as session_obj:- fps_manager = FPSCamCounter(self._cam_manager.available_webcam_ids)- fps_manager.start_all()- try:- should_continue = True- while should_continue:- for response in session_obj:- cv_cam = response.cv_cam- writer = response.writer- charuco_frame_payload = self._board_detection_object.detect_charuco_board_in_camera_stream(cv_cam)- current_webcam_id = cv_cam.webcam_id_as_str-- if not charuco_frame_payload:- continue-- if save_video:- writer.write(charuco_frame_payload.raw_frame_payload)-- if post_processed_frame_cb:- undistorted_annotated_image = post_processed_frame_cb(current_webcam_id,- charuco_frame_payload)- charuco_frame_payload.annotated_frame_image = undistorted_annotated_image-- fps_manager.increment_frame_processed_for(current_webcam_id)- if show_window:- should_continue = show_cam_window(- current_webcam_id, charuco_frame_payload.annotated_frame_image, fps_manager- )- except:- logger.error("Printing traceback")- traceback.print_exc()- finally:- for response in session_obj:- cv_cam = response.cv_cam- writer = response.writer- options = SaveOptions(- writer_dir=Path().joinpath(- cv_cam.session_writer_base_path,- "board_detection",- f"webcam_{cv_cam.webcam_id_as_str}",- ),- fps=fps_manager.current_fps_for(cv_cam.webcam_id_as_str),- frame_width=cv_cam.get_frame_width(),- frame_height=cv_cam.get_frame_height(),- )- writer.save(options)- logger.info(f"Destroy window {cv_cam.webcam_id_as_str}")- cv2.destroyWindow(cv_cam.webcam_id_as_str)- cv2.waitKey(1)-- def _calibration_work(self, webcam_id: str, charuco_frame_payload: CharucoFramePayload):- elapsed_time = (charuco_frame_payload.raw_frame_payload.timestamp / 1e9) - self._start_time+ def process_incoming_frame(self, charuco_frame_payload: CharucoFramePayload):if not charuco_frame_payload.charuco_view_data.full_board_found and self._current_calibration.camera_matrix is None:return charuco_frame_payload.annotated_frame_image@@ -188,15 +78,9 @@ class CameraCalibration:self._current_calibration.camera_matrix,self._current_calibration.lens_distortion_coefficients,)-- print(- f"found frame from camera {webcam_id} at timestamp {elapsed_time:.3f} - reprojection error: {self._current_calibration.reprojection_error:.3f} - num charuco views:{len(self._all_charuco_views)}")return undistorted_image- def estimate_lens_distortion(self,- new_charuco_view: CharucoViewData,- max_iterations=10,- max_charuco_board_views=5) -> SingleCameraCalibrationData:+ def estimate_lens_distortion(self, new_charuco_view: CharucoViewData):"""adapted from - https://mecaruco2.readthedocs.io/en/latest/notebooks_rst/Aruco/sandbox/ludovic/aruco_calibration_rotation.html@@ -219,14 +103,16 @@ class CameraCalibration:image_width = new_charuco_view.image_widthimage_height = new_charuco_view.image_height- default_calibration = self.get_default_calibration(new_charuco_view,- number_of_distortion_coefficients=5)+ default_calibration = LensDistortionCalibrationData(image_width, image_height)- num_charuco_views_per_combo = 10+ num_charuco_views_per_combo = 5if self._num_charuco_views < num_charuco_views_per_combo:list_of_combos_of_charuco_views = [self._all_charuco_views]else:- list_of_combos_of_charuco_views = self.generate_combos_of_charuco_views(new_charuco_view)+ list_of_combos_of_charuco_views = self.generate_combos_of_charuco_views(new_charuco_view,+ num_combos_to_generate=10,+ num_views_per_combo=num_charuco_views_per_combo,+ )# flag definitions -> https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html#ga3207604e4b1a1758aa66acb6ed5aa65dflags = (cv2.CALIB_USE_INTRINSIC_GUESS +@@ -261,11 +147,11 @@ class CameraCalibration:if self.show_calibration_diagnostics_visualizer:self.calibration_diagnostics_visualizer.update_reprojection_error_subplot(this_reprojection_error,- current_best_reprojection_error)+ current_best_reprojection_error)if this_reprojection_error < current_best_reprojection_error:- this_calibration = SingleCameraCalibrationData(+ this_calibration = LensDistortionCalibrationData(reprojection_error=this_reprojection_error,camera_matrix=this_camera_matrix,lens_distortion_coefficients=these_lens_distortion_coefficients,@@ -327,7 +213,7 @@ class CameraCalibration:def is_this_calibration_valid(self, this_calibration):is_valid = False- if not self.lens_distortion_is_monotonic(this_calibration):+ if not self.lens_distortion_is_monotonic(this_calibration): #not currently working - returns True every timereturn Falseif self.check_if_too_many_invalid_pixels(this_calibration):@@ -381,12 +267,11 @@ class CameraCalibration:if self.show_calibration_diagnostics_visualizer:self.calibration_diagnostics_visualizer.update_image_point_remapping_subplot(original_points_x,- original_points_y,- distorted_points_x,- distorted_points_y)+ original_points_y,+ distorted_points_x,+ distorted_points_y)return True-def check_if_too_many_invalid_pixels(self, this_calibration, how_many_is_too_many=.1) -> bool:"""create new camera matrix that will show full undistored image with black pixels in spots with no data.@@ -423,28 +308,6 @@ class CameraCalibration:return undistorted_image- @staticmethod- def get_default_calibration(charuco_view: CharucoViewData, number_of_distortion_coefficients=4):- image_width = charuco_view.image_width- image_height = charuco_view.image_height-- initial_camera_matrix = np.array([[float(image_width), 0., image_width / 2.],- [0., float(image_width), image_height / 2.],- [0., 0., 1.]])- initial_lens_distortion_coefficients = np.zeros((number_of_distortion_coefficients, 1))-- return SingleCameraCalibrationData(- camera_matrix=initial_camera_matrix,- lens_distortion_coefficients=initial_lens_distortion_coefficients,- image_width=image_width,- image_height=image_height,- )-if __name__ == "__main__":- print('start main')-- a = CameraCalibration()- a.perform_calibration()- if a.show_calibration_diagnostics_visualizer:- a.calibration_diagnostics_visualizer.close()+ passdiff --git a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.pyindex ca63c002..9522800c 100644--- a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py+++ b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py@@ -9,7 +9,7 @@ import mediapipe as mpimport numpy as npfrom src.cameras.capture.frame_payload import FramePayload-from src.cameras.multicam_manager.cv_camera_manager import CVCameraManager+from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManagerfrom src.cameras.persistence.video_writer.video_writer import SaveOptionsfrom src.config.data_paths import create_home_data_directoryfrom src.core_processor.fps.fps_counter import FPSCamCounter@@ -32,7 +32,7 @@ class ProcessResponse:class MediapipeSkeletonDetection:- def __init__(self, cam_manager: CVCameraManager):+ def __init__(self, cam_manager: OpenCVCameraManager):self._cam_manager = cam_managerasync def process_as_frame_loop(self, webcam_id, cb, model_complexity: int = 1):@@ -112,8 +112,8 @@ class MediapipeSkeletonDetection:f"webcam_{cv_cam.webcam_id_as_str}",),fps=fps_manager.current_fps_for(cv_cam.webcam_id_as_str),- frame_width=cv_cam.get_frame_width(),- frame_height=cv_cam.get_frame_height(),+ frame_width=cv_cam.image_width(),+ frame_height=cv_cam.image_height(),)response.writer.save(options)logger.info(f"Destroy window {cv_cam.webcam_id_as_str}")@@ -164,4 +164,4 @@ class MediapipeSkeletonDetection:if __name__ == "__main__":create_home_data_directory()- MediapipeSkeletonDetection(CVCameraManager()).process()+ MediapipeSkeletonDetection(OpenCVCameraManager()).process()diff --git a/src/core_processor/session/session_manager.py b/src/core_processor/session/session_manager.pynew file mode 100644index 00000000..e7158fab--- /dev/null+++ b/src/core_processor/session/session_manager.py@@ -0,0 +1,128 @@+import logging+import time+import traceback+from pathlib import Path++import cv2++from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager+from src.cameras.persistence.video_writer.save_options import SaveOptions+from src.core_processor.camera_calibration.camera_calibrator import CameraCalibrator+from src.core_processor.fps.fps_counter import FPSCamCounter+from src.core_processor.show_cam_window import show_cam_window++logger = logging.getLogger(__name__)+++class SessionManager:+ def __init__(self,+ opencv_camera_manager: OpenCVCameraManager = OpenCVCameraManager(),+ ):+ self._open_cv_camera_manager = opencv_camera_manager+ self._camera_calibrator = CameraCalibrator()+ self._start_time = time.time()++ async def process_by_cam_id(self, webcam_id: str, cb):+ with self._open_cv_camera_manager.start_capture_session_single_cam(+ webcam_id=webcam_id,+ ) as session_obj:+ fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)+ fps_manager.start_all()+ cv_cam = session_obj.cv_cam+ writer = session_obj.writer+ try:+ while True:+ if not cv_cam.is_capturing_frames:+ return+ charuco_frame_payload = self._board_detection_object.detect_charuco_board_in_camera_stream(cv_cam)+ if cb and charuco_frame_payload.annotated_frame_image is not None:+ writer.write(charuco_frame_payload.raw_frame_payload)+ await cb(charuco_frame_payload.annotated_frame_image)+ except Exception as e:+ logger.error("Printing traceback")+ traceback.print_exc()+ raise e+ finally:+ writer = session_obj.writer+ options = SaveOptions(+ writer_dir=Path().joinpath(+ cv_cam.session_writer_base_path,+ "board_detection",+ f"webcam_{cv_cam.webcam_id_as_str}",+ ),+ fps=fps_manager.current_fps_for(cv_cam.webcam_id_as_str),+ frame_width=cv_cam.image_width(),+ frame_height=cv_cam.image_height(),+ )+ writer.save(options)++ def run(+ self,+ calibrate_cameras=True,+ show_camera_views_in_windows=True,+ save_video=True,+ ):+ """+ Opens Cameras using OpenCV and begins image processing for charuco board+ If return images is true, the images are returned to the caller+ """+ # if its already capturing frames, this is a no-op++ with self._open_cv_camera_manager.start_capture_session_all_cams() as session_obj:+ fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)+ fps_manager.start_all()+ try:+ should_continue = True+ while should_continue:+ for response in session_obj:+ this_open_cv_camera = response.cv_cam+ this_video_writer_object = response.writer++ this_webcam_id = this_open_cv_camera.webcam_id_as_str+ this_cam_latest_frame = this_open_cv_camera.latest_frame++ if this_cam_latest_frame.image is None:+ continue++ image_to_display = this_cam_latest_frame.image++ if save_video:+ this_video_writer_object.write(this_cam_latest_frame)++ if calibrate_cameras:+ undistorted_annotated_image = self._camera_calibrator.calibrate(this_cam_latest_frame)+ image_to_display = undistorted_annotated_image++ fps_manager.increment_frame_processed_for(this_webcam_id)+ if show_camera_views_in_windows:+ should_continue = show_cam_window(+ this_webcam_id, image_to_display, fps_manager+ )+ except:+ logger.error("Printing traceback")+ traceback.print_exc()+ finally:+ for response in session_obj:+ this_open_cv_camera = response.cv_cam+ this_video_writer_object = response.writer+ options = SaveOptions(+ writer_dir=Path().joinpath(+ this_open_cv_camera.session_writer_base_path,+ "board_detection",+ f"webcam_{this_open_cv_camera.webcam_id_as_str}",+ ),+ fps=fps_manager.current_fps_for(this_open_cv_camera.webcam_id_as_str),+ frame_width=this_open_cv_camera.image_width,+ frame_height=this_open_cv_camera.image_height,+ )+ this_video_writer_object.save(options)+ logger.info(f"Destroy window {this_open_cv_camera.webcam_id_as_str}")+ cv2.destroyWindow(this_open_cv_camera.webcam_id_as_str)+ cv2.waitKey(1)+++if __name__ == "__main__":+ print('start main')++ this_session = SessionManager()+ this_session.run(calibrate_cameras=False)