Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

refactor to run everything through `CameraManager`

Jone7bbb4d5fad0
diff --git a/src/api/routes/camera/cam_data_ws.py b/src/api/routes/camera/cam_data_ws.py
index f2eb25ea..f19d1643 100644
--- a/src/api/routes/camera/cam_data_ws.py
+++ b/src/api/routes/camera/cam_data_ws.py
@@ -6,7 +6,7 @@ from fastapi import APIRouter, WebSocket
from src.api.services.board_detect_service import BoardDetectService
from src.api.services.mediapipe_detect_service import MediapipeSkeletonDetectionService
-from src.cameras.multicam_manager.cv_camera_manager import CVCameraManager
+from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager
logger = logging.getLogger(__name__)
@@ -50,7 +50,7 @@ async def skeleton_detection_as_ws(
await web_socket.send_bytes(frame.tobytes())
try:
- await MediapipeSkeletonDetectionService(CVCameraManager()).run_as_loop(
+ await MediapipeSkeletonDetectionService(OpenCVCameraManager()).run_as_loop(
webcam_id=webcam_id, cb=websocket_send, model_complexity=model_complexity
)
except:
@@ -64,5 +64,5 @@ async def begin_mediapipe_skeleton_detection(model_complexity: int):
"""
model_complexity can be 1 (faster, less accurate) or 2 (slower, more accurate)
"""
- service = MediapipeSkeletonDetectionService(CVCameraManager())
+ service = MediapipeSkeletonDetectionService(OpenCVCameraManager())
service.run(model_complexity)
diff --git a/src/api/services/board_detect_service.py b/src/api/services/board_detect_service.py
index 993e7f74..1887791a 100644
--- a/src/api/services/board_detect_service.py
+++ b/src/api/services/board_detect_service.py
@@ -1,7 +1,7 @@
from typing import NamedTuple
from src.cameras.multicam_manager import CVCameraManager
-from src.core_processor.board_detection.board_detection import BoardDetection
+from src.core_processor.board_detection.board_detection import BoardDetector
class ImageResponse(NamedTuple):
@@ -14,7 +14,7 @@ class BoardDetectService:
self._cam_manager = cam_manager
async def run(self):
- return BoardDetection(self._cam_manager).process()
+ return BoardDetector(self._cam_manager).process()
async def run_detection_on_cam_id(self, webcam_id: str, cb=None):
- await BoardDetection(self._cam_manager).process_by_cam_id(webcam_id, cb)
+ await BoardDetector(self._cam_manager).process_by_cam_id(webcam_id, cb)
diff --git a/src/api/services/mediapipe_detect_service.py b/src/api/services/mediapipe_detect_service.py
index 87b4b234..e03c85a6 100644
--- a/src/api/services/mediapipe_detect_service.py
+++ b/src/api/services/mediapipe_detect_service.py
@@ -1,11 +1,11 @@
-from src.cameras.multicam_manager.cv_camera_manager import CVCameraManager
+from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager
from src.core_processor.mediapipe_skeleton_detection.mediapipe_skeleton_detection import (
MediapipeSkeletonDetection,
)
class MediapipeSkeletonDetectionService:
- def __init__(self, cam_manager: CVCameraManager):
+ def __init__(self, cam_manager: OpenCVCameraManager):
self._cam_manager = cam_manager
def run(self, model_complexity: int):
diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.py
index 51c4cf8f..ecceb53f 100644
--- a/src/cameras/capture/opencv_camera/opencv_camera.py
+++ b/src/cameras/capture/opencv_camera/opencv_camera.py
@@ -9,6 +9,7 @@ from pydantic import BaseModel
from src.cameras.capture.frame_payload import FramePayload
from src.cameras.capture.opencv_camera.frame_grabber import FrameThread
+from src.core_processor.camera_calibration.camera_calibrator import CameraCalibrationData
logger = logging.getLogger(__name__)
@@ -38,6 +39,22 @@ class OpenCVCamera:
self._name = f"Camera {self._config.webcam_id}"
self._opencv_video_capture_object: cv2.VideoCapture = None
self._running_thread: FrameThread = None
+ self._camera_calibration_data = None
+
+ @property
+ def camera_calibration_data(self):
+ if self._camera_calibration_data is not None:
+ return self._camera_calibration_data
+
+ if self._camera_calibration_data is None and self.image_width is not None:
+ return self.camera_calibration_data(CameraCalibrationData(self.image_width, self.image_height))
+
+ logger.warning('Can\'t request a cameras calibration info until the video capture is initialized and the image_width and image_height are known')
+ return None
+
+ @camera_calibration_data.setter
+ def camera_calibration_data(self, camera_calibration_data: CameraCalibrationData):
+ self._camera_calibration_data = camera_calibration_data
@property
def webcam_id_as_str(self):
@@ -89,6 +106,7 @@ class OpenCVCamera:
logger.debug(f"Camera found at port number {self._config.webcam_id}")
fps_input_stream = int(self._opencv_video_capture_object.get(5))
logger.debug("FPS of webcam hardware/input stream: {}".format(fps_input_stream))
+
return success
def start_frame_capture(self):
@@ -108,15 +126,23 @@ class OpenCVCamera:
webcam_id=self.webcam_id_as_str,
get_next_frame=self.get_next_frame,
save_video=self._config.save_video,
- frame_width=self.get_frame_width(),
- frame_height=self.get_frame_height(),
+ frame_width=self.image_width,
+ frame_height=self.image_height,
)
- def get_frame_width(self):
- return int(self._opencv_video_capture_object.get(3))
+ @property
+ def image_width(self):
+ try:
+ return int(self._opencv_video_capture_object.get(3))
+ except:
+ return None
- def get_frame_height(self):
- return int(self._opencv_video_capture_object.get(4))
+ @property
+ def image_height(self):
+ try:
+ return int(self._opencv_video_capture_object.get(4))
+ except:
+ return None
def _apply_configuration(self):
# set camera stream parameters
diff --git a/src/cameras/detection/cam_detection.py b/src/cameras/detection/cam_detection.py
index d51fdad0..aaaab6c7 100644
--- a/src/cameras/detection/cam_detection.py
+++ b/src/cameras/detection/cam_detection.py
@@ -3,7 +3,7 @@ import platform
import cv2
-from src.cameras.detection.models import FoundCamerasResponse, RawCamera
+from src.cameras.detection.models import FoundCamerasResponse
CAM_CHECK_NUM = 20
@@ -15,27 +15,24 @@ class DetectPossibleCameras:
cv2_backend = self._determine_backend()
cams_to_use_list = []
- for cam_id in range(CAM_CHECK_NUM):
- cap = cv2.VideoCapture(cam_id, cv2_backend)
+ for this_usb_port_number in range(CAM_CHECK_NUM):
+ cap = cv2.VideoCapture(this_usb_port_number, cv2_backend)
success, image = cap.read()
if success:
try:
- cams_to_use_list.append(
- RawCamera(
- webcam_id=cam_id,
- )
- )
+ cams_to_use_list.append(this_usb_port_number)
finally:
cap.release()
return FoundCamerasResponse(
- camera_found_count=len(cams_to_use_list),
- cams_to_use=cams_to_use_list,
+ number_of_cameras_found=len(cams_to_use_list),
+ list_of_usb_port_numbers_with_cameras_attached=cams_to_use_list,
cv2_backend=cv2_backend,
)
- def _determine_backend(self):
+ @staticmethod
+ def _determine_backend():
if platform.system() == "Windows":
return cv2.CAP_DSHOW
else:
diff --git a/src/cameras/detection/models.py b/src/cameras/detection/models.py
index dee5c0f8..d96b3a60 100644
--- a/src/cameras/detection/models.py
+++ b/src/cameras/detection/models.py
@@ -3,11 +3,7 @@ from typing import List
from pydantic import BaseModel
-class RawCamera(BaseModel):
- webcam_id: str
-
-
class FoundCamerasResponse(BaseModel):
- camera_found_count: int
- cams_to_use: List[RawCamera]
+ number_of_cameras_found: int
+ list_of_usb_port_numbers_with_cameras_attached: List[int]
cv2_backend: int
diff --git a/src/cameras/multicam_manager/cv_camera_manager.py b/src/cameras/multicam_manager/cv_camera_manager.py
index 1ac6a432..42d13006 100644
--- a/src/cameras/multicam_manager/cv_camera_manager.py
+++ b/src/cameras/multicam_manager/cv_camera_manager.py
@@ -9,10 +9,11 @@ from src.api.services.user_config import UserConfigService
from src.cameras.detection.cam_singleton import get_or_create_cams
from src.cameras.capture.opencv_camera.opencv_camera import OpenCVCamera
from src.cameras.persistence.video_writer.video_writer import VideoWriter
+from src.core_processor.camera_calibration.camera_calibrator import CameraCalibrationData
logger = logging.getLogger(__name__)
-_cv_cams = None
+_open_cv_cameras = None
class CamAndWriterResponse(BaseModel):
@@ -23,41 +24,42 @@ class CamAndWriterResponse(BaseModel):
arbitrary_types_allowed = True
-class CVCameraManager:
+class OpenCVCameraManager:
def __init__(self):
self._config_service = UserConfigService()
self._detected_cams_data = get_or_create_cams()
- global _cv_cams
- if _cv_cams is None:
+ global _open_cv_cameras
+ if _open_cv_cameras is None:
logger.info("Creating cams.")
# we create the _cv_cams /once/, and reuse it for the lifetime of the session
- _cv_cams = self._create_opencv_cams()
- self._cv_cams = _cv_cams
+ _open_cv_cameras = self._create_opencv_cameras(calibrate_cameras=True)
+ self._open_cv_camera_objects = _open_cv_cameras
else:
logger.info("Reusing already created resources cam resources.")
- self._cv_cams = _cv_cams
+ self._open_cv_camera_objects = _open_cv_cameras
@property
def available_webcam_ids(self):
- return [cv_cam.webcam_id_as_str for cv_cam in self._cv_cams]
+ return [cv_cam.webcam_id_as_str for cv_cam in self._open_cv_camera_objects]
@property
- def cv_cams(self):
- return self._cv_cams
+ def open_cv_cameras(self):
+ return self._open_cv_camera_objects
def cv_cam_by_id(self, webcam_id: str):
- for cam in self._cv_cams:
+ for cam in self._open_cv_camera_objects:
if cam.webcam_id_as_str == str(webcam_id):
return cam
return None
- def _create_opencv_cams(self):
- raw_webcam_obj = self._detected_cams_data.cams_to_use
- cv_cams: List[OpenCVCamera] = []
- for webcam in raw_webcam_obj:
- single_config = self._config_service.webcam_config_by_id(webcam.webcam_id)
- cv_cams.append(OpenCVCamera(config=single_config))
- return cv_cams
+ def _create_opencv_cameras(self, calibrate_cameras=True):
+ list_of_usb_port_numbers_with_cameras_attached = self._detected_cams_data.list_of_usb_port_numbers_with_cameras_attached
+ open_cv_cameras: List[OpenCVCamera] = []
+ for this_port_number in list_of_usb_port_numbers_with_cameras_attached:
+ single_camera_config = self._config_service.webcam_config_by_id(this_port_number)
+ this_opencv_camera = OpenCVCamera(config=single_camera_config)
+ open_cv_cameras.append(this_opencv_camera)
+ return open_cv_cameras
@contextmanager
def start_capture_session_single_cam(
@@ -94,7 +96,7 @@ class CVCameraManager:
def _start_frame_capture_all_cams(self) -> Dict[str, VideoWriter]:
d = {}
- for cv_cam in self._cv_cams:
+ for cv_cam in self._open_cv_camera_objects:
cv_cam.connect()
cv_cam.start_frame_capture()
d[cv_cam.webcam_id_as_str] = VideoWriter()
@@ -103,7 +105,7 @@ class CVCameraManager:
def _start_frame_capture_on_cam_id(self, webcam_id: str) -> VideoWriter:
filtered_cams = list(
- filter(lambda c: c.webcam_id_as_str == str(webcam_id), self._cv_cams)
+ filter(lambda c: c.webcam_id_as_str == str(webcam_id), self._open_cv_camera_objects)
)
assert (
len(filtered_cams) == 1
@@ -114,9 +116,9 @@ class CVCameraManager:
return VideoWriter()
def _stop_frame_capture_all_cams(self):
- for cv_cam in self._cv_cams:
+ for cv_cam in self._open_cv_camera_objects:
cv_cam.stop_frame_capture()
def _close_all_cameras(self):
- for cv_cam in self._cv_cams:
+ for cv_cam in self._open_cv_camera_objects:
cv_cam.close()
diff --git a/src/core_processor/board_detection/board_detection.py b/src/core_processor/board_detection/board_detection.py
index 093df174..ad972b87 100644
--- a/src/core_processor/board_detection/board_detection.py
+++ b/src/core_processor/board_detection/board_detection.py
@@ -1,26 +1,16 @@
import logging
-import time
-import traceback
from dataclasses import dataclass
-from pathlib import Path
-from types import FunctionType
-from typing import Any, Callable, Optional
import cv2
import numpy as np
from src.cameras.capture.frame_payload import FramePayload
-from src.cameras.multicam_manager.cv_camera_manager import CVCameraManager
-from src.cameras.persistence.video_writer.video_writer import SaveOptions
from src.core_processor.board_detection.charuco_constants import aruco_marker_dict, charuco_board_object, \
number_of_charuco_corners
from src.core_processor.board_detection.detect_charuco_board import detect_charuco_board, CharucoViewData
from src.core_processor.board_detection.charuco_image_annotator import (
annotate_image_with_charuco_data,
)
-from src.core_processor.fps.fps_counter import FPSCamCounter
-from src.core_processor.show_cam_window import show_cam_window
-from src.core_processor.utils.image_fps_writer import write_fps_to_image
logger = logging.getLogger(__name__)
@@ -32,9 +22,9 @@ class CharucoFramePayload:
charuco_view_data: CharucoViewData
-class BoardDetection:
+class BoardDetector:
@staticmethod
- def detect_charuco_board_in_camera_stream(cv_cam)->CharucoFramePayload:
+ def detect_charuco_board_in_camera_stream(cv_cam) -> CharucoFramePayload:
raw_frame_payload = cv_cam.latest_frame
if not raw_frame_payload.success:
@@ -59,6 +49,7 @@ class BoardDetection:
raw_frame_payload=raw_frame_payload,
charuco_view_data=charuco_view_data
)
+
@staticmethod
def detect_charuco_board(image) -> CharucoViewData:
"""
diff --git a/src/core_processor/camera_calibration/camera_calibrator.py b/src/core_processor/camera_calibration/camera_calibrator.py
new file mode 100644
index 00000000..7847db55
--- /dev/null
+++ b/src/core_processor/camera_calibration/camera_calibrator.py
@@ -0,0 +1,37 @@
+from dataclasses import dataclass
+from typing import List
+
+import numpy as np
+
+from src.cameras.capture.frame_payload import FramePayload
+from src.core_processor.camera_calibration.lens_distortion_calibrator import LensDistortionCalibrator, \
+ LensDistortionCalibrationData
+
+
+@dataclass
+class CameraCalibrationData:
+ image_width: int
+ image_height: int
+ lens_distortion_calibration_data: LensDistortionCalibrationData = None
+ camera_translation_relative_to_camera0: np.ndarray = np.zeros((3, 1))
+ camera_rotation_relative_to_camera0: np.ndarray = np.zeros((3, 1))
+
+ def __init__(self, image_width, image_height):
+ self.image_width = image_width
+ self.image_height = image_height
+ self.lens_distortion_calibration_data = LensDistortionCalibrationData(self.image_width,
+ self.image_height)
+
+
+class CameraCalibrator:
+ def __init__(self,
+ lens_distortion_calibrator: LensDistortionCalibrator = LensDistortionCalibrator()
+ ):
+ self._lens_distortion_calibrator = lens_distortion_calibrator
+
+ def calibrate(self, this_frame: FramePayload):
+ self.calibrate_lens_distortion(this_frame)
+ # something something cameras 6DoF
+
+ def calibrate_lens_distortion(self, this_frame):
+ self._lens_distortion_calibrator.process_incoming_frame(this_frame)
diff --git a/src/core_processor/camera_calibration/camera_calibration.py b/src/core_processor/camera_calibration/lens_distortion_calibrator.py
similarity index 65%
rename from src/core_processor/camera_calibration/camera_calibration.py
rename to src/core_processor/camera_calibration/lens_distortion_calibrator.py
index 8ead5208..dd56253c 100644
--- a/src/core_processor/camera_calibration/camera_calibration.py
+++ b/src/core_processor/camera_calibration/lens_distortion_calibrator.py
@@ -1,49 +1,44 @@
-import logging
-import traceback
from dataclasses import dataclass
-import time
-from pathlib import Path
-from typing import Callable
import cv2
-import matplotlib.pyplot as plt
-import pyqtgraph as pg
import numpy as np
from rich import print
-from src.cameras.multicam_manager.cv_camera_manager import CVCameraManager
-from src.cameras.persistence.video_writer.save_options import SaveOptions
-from src.core_processor.board_detection.board_detection import BoardDetection, CharucoFramePayload
+from src.core_processor.board_detection.board_detection import BoardDetector, CharucoFramePayload
from src.core_processor.board_detection.charuco_constants import charuco_board_object
from src.core_processor.board_detection.detect_charuco_board import CharucoViewData
from src.core_processor.camera_calibration.calibration_diagnostics_visualizer import CalibrationDiagnosticsVisualizer
-from src.core_processor.fps.fps_counter import FPSCamCounter
-from src.core_processor.show_cam_window import show_cam_window
-logger = logging.getLogger(__name__)
@dataclass
-class SingleCameraCalibrationData:
+class LensDistortionCalibrationData:
+ image_width: int
+ image_height: int
reprojection_error: float = None
camera_matrix: np.ndarray = None
+ number_of_lens_distortion_coefficients: int = 4
lens_distortion_coefficients: np.ndarray = None
- image_width: int = None
- image_height: int = None
rotation_vectors_of_the_board: np.ndarray = None
translation_vectors_of_the_board: np.ndarray = None
lens_distortion_std_dev: float = None
camera_location_std_dev: float = None
+ def __post_init__(self):
+ if self.camera_matrix is None:
+ self.camera_matrix = np.array([[float(self.image_width), 0., self.image_width / 2.],
+ [0., float(self.image_width), self.image_height / 2.],
+ [0., 0., 1.]])
-class CameraCalibration:
+ if self.lens_distortion_coefficients is None:
+ self.lens_distortion_coefficients = np.zeros((self.number_of_lens_distortion_coefficients, 1))
+
+
+class LensDistortionCalibrator:
def __init__(self,
- board_detection: BoardDetection = BoardDetection(),
- cam_manager: CVCameraManager = CVCameraManager(),
+ board_detection_object: BoardDetector = BoardDetector(),
show_calibration_diagnostics_visualizer=True, ):
- self._board_detection_object = board_detection
- self._cam_manager = cam_manager
- self._current_calibration = SingleCameraCalibrationData()
- self._start_time = time.time()
+ self._board_detection_object = board_detection_object
+ self._current_calibration: LensDistortionCalibrationData = None
self._all_charuco_views = []
self._num_charuco_views = 0
self._best_combo_of_charuco_views = []
@@ -53,112 +48,7 @@ class CameraCalibration:
self.calibration_diagnostics_visualizer = CalibrationDiagnosticsVisualizer()
- async def process_by_cam_id(self, webcam_id: str, cb):
- with self._cam_manager.start_capture_session_single_cam(
- webcam_id=webcam_id,
- ) as session_obj:
- fps_manager = FPSCamCounter(self._cam_manager.available_webcam_ids)
- fps_manager.start_all()
- cv_cam = session_obj.cv_cam
- writer = session_obj.writer
- try:
- while True:
- if not cv_cam.is_capturing_frames:
- return
- charuco_frame_payload = self._board_detection_object.detect_charuco_board_in_camera_stream(cv_cam)
- if cb and charuco_frame_payload.annotated_frame_image is not None:
- writer.write(charuco_frame_payload.raw_frame_payload)
- await cb(charuco_frame_payload.annotated_frame_image)
- except Exception as e:
- logger.error("Printing traceback")
- traceback.print_exc()
- raise e
- finally:
- writer = session_obj.writer
- options = SaveOptions(
- writer_dir=Path().joinpath(
- cv_cam.session_writer_base_path,
- "board_detection",
- f"webcam_{cv_cam.webcam_id_as_str}",
- ),
- fps=fps_manager.current_fps_for(cv_cam.webcam_id_as_str),
- frame_width=cv_cam.get_frame_width(),
- frame_height=cv_cam.get_frame_height(),
- )
- writer.save(options)
-
-
- def perform_calibration(self):
- self.process(
- show_window=True,
- save_video=False,
- post_processed_frame_cb=self._calibration_work,
- )
-
- def process(
- self,
- post_processed_frame_cb: Callable[[str, CharucoFramePayload], None] = None,
- show_window=True,
- save_video=True,
- ):
- """
- Opens Cameras using OpenCV and begins image processing for charuco board
- If return images is true, the images are returned to the caller
- """
- # if its already capturing frames, this is a no-op
-
- with self._cam_manager.start_capture_session_all_cams() as session_obj:
- fps_manager = FPSCamCounter(self._cam_manager.available_webcam_ids)
- fps_manager.start_all()
- try:
- should_continue = True
- while should_continue:
- for response in session_obj:
- cv_cam = response.cv_cam
- writer = response.writer
- charuco_frame_payload = self._board_detection_object.detect_charuco_board_in_camera_stream(cv_cam)
- current_webcam_id = cv_cam.webcam_id_as_str
-
- if not charuco_frame_payload:
- continue
-
- if save_video:
- writer.write(charuco_frame_payload.raw_frame_payload)
-
- if post_processed_frame_cb:
- undistorted_annotated_image = post_processed_frame_cb(current_webcam_id,
- charuco_frame_payload)
- charuco_frame_payload.annotated_frame_image = undistorted_annotated_image
-
- fps_manager.increment_frame_processed_for(current_webcam_id)
- if show_window:
- should_continue = show_cam_window(
- current_webcam_id, charuco_frame_payload.annotated_frame_image, fps_manager
- )
- except:
- logger.error("Printing traceback")
- traceback.print_exc()
- finally:
- for response in session_obj:
- cv_cam = response.cv_cam
- writer = response.writer
- options = SaveOptions(
- writer_dir=Path().joinpath(
- cv_cam.session_writer_base_path,
- "board_detection",
- f"webcam_{cv_cam.webcam_id_as_str}",
- ),
- fps=fps_manager.current_fps_for(cv_cam.webcam_id_as_str),
- frame_width=cv_cam.get_frame_width(),
- frame_height=cv_cam.get_frame_height(),
- )
- writer.save(options)
- logger.info(f"Destroy window {cv_cam.webcam_id_as_str}")
- cv2.destroyWindow(cv_cam.webcam_id_as_str)
- cv2.waitKey(1)
-
- def _calibration_work(self, webcam_id: str, charuco_frame_payload: CharucoFramePayload):
- elapsed_time = (charuco_frame_payload.raw_frame_payload.timestamp / 1e9) - self._start_time
+ def process_incoming_frame(self, charuco_frame_payload: CharucoFramePayload):
if not charuco_frame_payload.charuco_view_data.full_board_found and self._current_calibration.camera_matrix is None:
return charuco_frame_payload.annotated_frame_image
@@ -188,15 +78,9 @@ class CameraCalibration:
self._current_calibration.camera_matrix,
self._current_calibration.lens_distortion_coefficients,
)
-
- print(
- f"found frame from camera {webcam_id} at timestamp {elapsed_time:.3f} - reprojection error: {self._current_calibration.reprojection_error:.3f} - num charuco views:{len(self._all_charuco_views)}")
return undistorted_image
- def estimate_lens_distortion(self,
- new_charuco_view: CharucoViewData,
- max_iterations=10,
- max_charuco_board_views=5) -> SingleCameraCalibrationData:
+ def estimate_lens_distortion(self, new_charuco_view: CharucoViewData):
"""
adapted from - https://mecaruco2.readthedocs.io/en/latest/notebooks_rst/Aruco/sandbox/ludovic/aruco_calibration_rotation.html
@@ -219,14 +103,16 @@ class CameraCalibration:
image_width = new_charuco_view.image_width
image_height = new_charuco_view.image_height
- default_calibration = self.get_default_calibration(new_charuco_view,
- number_of_distortion_coefficients=5)
+ default_calibration = LensDistortionCalibrationData(image_width, image_height)
- num_charuco_views_per_combo = 10
+ num_charuco_views_per_combo = 5
if self._num_charuco_views < num_charuco_views_per_combo:
list_of_combos_of_charuco_views = [self._all_charuco_views]
else:
- list_of_combos_of_charuco_views = self.generate_combos_of_charuco_views(new_charuco_view)
+ list_of_combos_of_charuco_views = self.generate_combos_of_charuco_views(new_charuco_view,
+ num_combos_to_generate=10,
+ num_views_per_combo=num_charuco_views_per_combo,
+ )
# flag definitions -> https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html#ga3207604e4b1a1758aa66acb6ed5aa65d
flags = (cv2.CALIB_USE_INTRINSIC_GUESS +
@@ -261,11 +147,11 @@ class CameraCalibration:
if self.show_calibration_diagnostics_visualizer:
self.calibration_diagnostics_visualizer.update_reprojection_error_subplot(this_reprojection_error,
- current_best_reprojection_error)
+ current_best_reprojection_error)
if this_reprojection_error < current_best_reprojection_error:
- this_calibration = SingleCameraCalibrationData(
+ this_calibration = LensDistortionCalibrationData(
reprojection_error=this_reprojection_error,
camera_matrix=this_camera_matrix,
lens_distortion_coefficients=these_lens_distortion_coefficients,
@@ -327,7 +213,7 @@ class CameraCalibration:
def is_this_calibration_valid(self, this_calibration):
is_valid = False
- if not self.lens_distortion_is_monotonic(this_calibration):
+ if not self.lens_distortion_is_monotonic(this_calibration): #not currently working - returns True every time
return False
if self.check_if_too_many_invalid_pixels(this_calibration):
@@ -381,12 +267,11 @@ class CameraCalibration:
if self.show_calibration_diagnostics_visualizer:
self.calibration_diagnostics_visualizer.update_image_point_remapping_subplot(original_points_x,
- original_points_y,
- distorted_points_x,
- distorted_points_y)
+ original_points_y,
+ distorted_points_x,
+ distorted_points_y)
return True
-
def check_if_too_many_invalid_pixels(self, this_calibration, how_many_is_too_many=.1) -> bool:
"""
create new camera matrix that will show full undistored image with black pixels in spots with no data.
@@ -423,28 +308,6 @@ class CameraCalibration:
return undistorted_image
- @staticmethod
- def get_default_calibration(charuco_view: CharucoViewData, number_of_distortion_coefficients=4):
- image_width = charuco_view.image_width
- image_height = charuco_view.image_height
-
- initial_camera_matrix = np.array([[float(image_width), 0., image_width / 2.],
- [0., float(image_width), image_height / 2.],
- [0., 0., 1.]])
- initial_lens_distortion_coefficients = np.zeros((number_of_distortion_coefficients, 1))
-
- return SingleCameraCalibrationData(
- camera_matrix=initial_camera_matrix,
- lens_distortion_coefficients=initial_lens_distortion_coefficients,
- image_width=image_width,
- image_height=image_height,
- )
-
if __name__ == "__main__":
- print('start main')
-
- a = CameraCalibration()
- a.perform_calibration()
- if a.show_calibration_diagnostics_visualizer:
- a.calibration_diagnostics_visualizer.close()
+ pass
diff --git a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py
index ca63c002..9522800c 100644
--- a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py
+++ b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py
@@ -9,7 +9,7 @@ import mediapipe as mp
import numpy as np
from src.cameras.capture.frame_payload import FramePayload
-from src.cameras.multicam_manager.cv_camera_manager import CVCameraManager
+from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager
from src.cameras.persistence.video_writer.video_writer import SaveOptions
from src.config.data_paths import create_home_data_directory
from src.core_processor.fps.fps_counter import FPSCamCounter
@@ -32,7 +32,7 @@ class ProcessResponse:
class MediapipeSkeletonDetection:
- def __init__(self, cam_manager: CVCameraManager):
+ def __init__(self, cam_manager: OpenCVCameraManager):
self._cam_manager = cam_manager
async def process_as_frame_loop(self, webcam_id, cb, model_complexity: int = 1):
@@ -112,8 +112,8 @@ class MediapipeSkeletonDetection:
f"webcam_{cv_cam.webcam_id_as_str}",
),
fps=fps_manager.current_fps_for(cv_cam.webcam_id_as_str),
- frame_width=cv_cam.get_frame_width(),
- frame_height=cv_cam.get_frame_height(),
+ frame_width=cv_cam.image_width(),
+ frame_height=cv_cam.image_height(),
)
response.writer.save(options)
logger.info(f"Destroy window {cv_cam.webcam_id_as_str}")
@@ -164,4 +164,4 @@ class MediapipeSkeletonDetection:
if __name__ == "__main__":
create_home_data_directory()
- MediapipeSkeletonDetection(CVCameraManager()).process()
+ MediapipeSkeletonDetection(OpenCVCameraManager()).process()
diff --git a/src/core_processor/session/session_manager.py b/src/core_processor/session/session_manager.py
new file mode 100644
index 00000000..e7158fab
--- /dev/null
+++ b/src/core_processor/session/session_manager.py
@@ -0,0 +1,128 @@
+import logging
+import time
+import traceback
+from pathlib import Path
+
+import cv2
+
+from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager
+from src.cameras.persistence.video_writer.save_options import SaveOptions
+from src.core_processor.camera_calibration.camera_calibrator import CameraCalibrator
+from src.core_processor.fps.fps_counter import FPSCamCounter
+from src.core_processor.show_cam_window import show_cam_window
+
+logger = logging.getLogger(__name__)
+
+
+class SessionManager:
+ def __init__(self,
+ opencv_camera_manager: OpenCVCameraManager = OpenCVCameraManager(),
+ ):
+ self._open_cv_camera_manager = opencv_camera_manager
+ self._camera_calibrator = CameraCalibrator()
+ self._start_time = time.time()
+
+ async def process_by_cam_id(self, webcam_id: str, cb):
+ with self._open_cv_camera_manager.start_capture_session_single_cam(
+ webcam_id=webcam_id,
+ ) as session_obj:
+ fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)
+ fps_manager.start_all()
+ cv_cam = session_obj.cv_cam
+ writer = session_obj.writer
+ try:
+ while True:
+ if not cv_cam.is_capturing_frames:
+ return
+ charuco_frame_payload = self._board_detection_object.detect_charuco_board_in_camera_stream(cv_cam)
+ if cb and charuco_frame_payload.annotated_frame_image is not None:
+ writer.write(charuco_frame_payload.raw_frame_payload)
+ await cb(charuco_frame_payload.annotated_frame_image)
+ except Exception as e:
+ logger.error("Printing traceback")
+ traceback.print_exc()
+ raise e
+ finally:
+ writer = session_obj.writer
+ options = SaveOptions(
+ writer_dir=Path().joinpath(
+ cv_cam.session_writer_base_path,
+ "board_detection",
+ f"webcam_{cv_cam.webcam_id_as_str}",
+ ),
+ fps=fps_manager.current_fps_for(cv_cam.webcam_id_as_str),
+ frame_width=cv_cam.image_width(),
+ frame_height=cv_cam.image_height(),
+ )
+ writer.save(options)
+
+ def run(
+ self,
+ calibrate_cameras=True,
+ show_camera_views_in_windows=True,
+ save_video=True,
+ ):
+ """
+ Opens Cameras using OpenCV and begins image processing for charuco board
+ If return images is true, the images are returned to the caller
+ """
+ # if its already capturing frames, this is a no-op
+
+ with self._open_cv_camera_manager.start_capture_session_all_cams() as session_obj:
+ fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)
+ fps_manager.start_all()
+ try:
+ should_continue = True
+ while should_continue:
+ for response in session_obj:
+ this_open_cv_camera = response.cv_cam
+ this_video_writer_object = response.writer
+
+ this_webcam_id = this_open_cv_camera.webcam_id_as_str
+ this_cam_latest_frame = this_open_cv_camera.latest_frame
+
+ if this_cam_latest_frame.image is None:
+ continue
+
+ image_to_display = this_cam_latest_frame.image
+
+ if save_video:
+ this_video_writer_object.write(this_cam_latest_frame)
+
+ if calibrate_cameras:
+ undistorted_annotated_image = self._camera_calibrator.calibrate(this_cam_latest_frame)
+ image_to_display = undistorted_annotated_image
+
+ fps_manager.increment_frame_processed_for(this_webcam_id)
+ if show_camera_views_in_windows:
+ should_continue = show_cam_window(
+ this_webcam_id, image_to_display, fps_manager
+ )
+ except:
+ logger.error("Printing traceback")
+ traceback.print_exc()
+ finally:
+ for response in session_obj:
+ this_open_cv_camera = response.cv_cam
+ this_video_writer_object = response.writer
+ options = SaveOptions(
+ writer_dir=Path().joinpath(
+ this_open_cv_camera.session_writer_base_path,
+ "board_detection",
+ f"webcam_{this_open_cv_camera.webcam_id_as_str}",
+ ),
+ fps=fps_manager.current_fps_for(this_open_cv_camera.webcam_id_as_str),
+ frame_width=this_open_cv_camera.image_width,
+ frame_height=this_open_cv_camera.image_height,
+ )
+ this_video_writer_object.save(options)
+ logger.info(f"Destroy window {this_open_cv_camera.webcam_id_as_str}")
+ cv2.destroyWindow(this_open_cv_camera.webcam_id_as_str)
+ cv2.waitKey(1)
+
+
+if __name__ == "__main__":
+ print('start main')
+
+ this_session = SessionManager()
+ this_session.run(calibrate_cameras=False)