Free Motion Capture for Everyone πβ¨
TypeScript
check in during twitch
Jone5398b9e86fb
diff --git a/requirements.txt b/requirements.txtindex d1095aea..3772c764 100644--- a/requirements.txt+++ b/requirements.txt@@ -1,17 +1,17 @@-aniposelib==0.4.3-h5py>=3.1.0-imutils~=0.5.4-matplotlib~=3.5.1-# mediapipe==0.8.8-opencv-contrib-python==4.5.5.62-pandas~=1.2.5-rich~=10.4.0-tqdm~=4.61.1-ruamel.yaml>=0.15.0-numpy>=1.19-typed-ast<1.5,>=1.4.0-moviepy ~=1.0.3+#aniposelib==0.4.3+#h5py>=3.1.0+#imutils~=0.5.4+#matplotlib~=3.5.1+## mediapipe==0.8.8+#opencv-contrib-python==4.5.5.62+#pandas~=1.2.5+#rich~=10.4.0+#tqdm~=4.61.1+#ruamel.yaml>=0.15.0+#typed-ast<1.5,>=1.4.0+#moviepy ~=1.0.3ipython+numpy>=1.19pydantic==1.9.0uvicorn==0.16.0fastapi==0.71.0diff --git a/src/api/services/board_detect_service.py b/src/api/services/board_detect_service.pyindex e87a7aad..4f4aca3e 100644--- a/src/api/services/board_detect_service.py+++ b/src/api/services/board_detect_service.py@@ -1,7 +1,7 @@from typing import NamedTuplefrom src.cameras.multicam_manager import CVCameraManager-from src.core_processor.camera_calibration.charuco_board_detection.board_detector import BoardDetector+from src.core_processor.camera_calibration.charuco_board_detection.charuco_board_detector import CharucoBoardDetectorclass ImageResponse(NamedTuple):@@ -14,7 +14,7 @@ class BoardDetectService:self._cam_manager = cam_managerasync def run(self):- return BoardDetector(self._cam_manager).process()+ return CharucoBoardDetector(self._cam_manager).process()async def run_detection_on_cam_id(self, webcam_id: str, cb=None):- await BoardDetector(self._cam_manager).process_by_cam_id(webcam_id, cb)+ await CharucoBoardDetector(self._cam_manager).process_by_cam_id(webcam_id, cb)diff --git a/src/core_processor/camera_calibration/calibration_dataclasses.py b/src/core_processor/camera_calibration/calibration_dataclasses.pyindex 0dae6848..85a19ea8 100644--- a/src/core_processor/camera_calibration/calibration_dataclasses.py+++ b/src/core_processor/camera_calibration/calibration_dataclasses.py@@ -3,9 +3,8 @@ from dataclasses import dataclassimport numpy as np-@dataclass-class LensDistortionCalibrationData:+class CameraCalibrationData:image_width: intimage_height: intreprojection_error: float = 1e9@@ -26,17 +25,4 @@ class LensDistortionCalibrationData:if self.lens_distortion_coefficients is None:self.lens_distortion_coefficients = np.zeros((self.number_of_lens_distortion_coefficients, 1))-@dataclass-class CameraCalibrationData:- image_width: int- image_height: int- lens_distortion_calibration_data: LensDistortionCalibrationData = None- camera_translation_relative_to_camera0: np.ndarray = np.zeros((3, 1))- camera_rotation_relative_to_camera0: np.ndarray = np.zeros((3, 1))-- def __init__(self, image_width, image_height):- self.image_width = image_width- self.image_height = image_height- self.lens_distortion_calibration_data = LensDistortionCalibrationData(self.image_width,- self.image_height)diff --git a/src/core_processor/camera_calibration/calibration_diagnostics_visualizer.py b/src/core_processor/camera_calibration/calibration_diagnostics_visualizer.pyindex 362b7b69..6a965497 100644--- a/src/core_processor/camera_calibration/calibration_diagnostics_visualizer.py+++ b/src/core_processor/camera_calibration/calibration_diagnostics_visualizer.py@@ -7,7 +7,7 @@ from pyqtgraph.Qt import QtWidgetsimport cv2-from src.core_processor.camera_calibration.calibration_dataclasses import LensDistortionCalibrationData+from src.core_processor.camera_calibration.calibration_dataclasses import CameraCalibrationData@@ -134,7 +134,7 @@ class CalibrationDiagnosticsVisualizer:self.reprojection_error_all_line.setData(self.reprojection_error_all)self.reprojection_error_current_best_line.setData(self.reprojection_error_current_best)- def update_calibration_text_overlay(self, calibration_data: LensDistortionCalibrationData):+ def update_calibration_text_overlay(self, calibration_data: CameraCalibrationData):lens_distortion_as_str = [f"{d[0]:.3f}" for d in calibration_data.lens_distortion_coefficients]calibration_text = f'reprojection error:{calibration_data.reprojection_error:.3f} \n {lens_distortion_as_str} \n {str(calibration_data.camera_matrix)}'self.calibration_text_widget.setText(calibration_text)diff --git a/src/core_processor/camera_calibration/camera_calibrator.py b/src/core_processor/camera_calibration/camera_calibrator.pyindex 48d2e5cf..fe0146b7 100644--- a/src/core_processor/camera_calibration/camera_calibrator.py+++ b/src/core_processor/camera_calibration/camera_calibrator.py@@ -1,21 +1,121 @@import loggingfrom dataclasses import dataclass+import cv2import numpy as np+from src.cameras.capture.dataclasses.frame_payload import FramePayloadfrom src.cameras.capture.opencv_camera.opencv_camera import OpenCVCamera-from src.core_processor.camera_calibration.lens_distortion_calibrator import LensDistortionCalibrator, \- LensDistortionCalibrationData+from src.core_processor.camera_calibration.calibration_dataclasses import CameraCalibrationData+from src.core_processor.camera_calibration.charuco_board_detection.charuco_board_detector import CharucoBoardDetector+from src.core_processor.camera_calibration.charuco_board_detection.charuco_image_annotator import \+ annotate_image_with_charuco_data+from src.core_processor.camera_calibration.charuco_board_detection.dataclasses.charuco_view_data import CharucoViewDatalogger = logging.getLogger(__name__)class CameraCalibrator:- def __init__(self):- self.lens_distortion_calibrator = LensDistortionCalibrator()+ def __init__(self,+ charuco_board_detector: CharucoBoardDetector = CharucoBoardDetector(),+ ):+ self._current_calibration = None+ self._charuco_board_detector = charuco_board_detectordef calibrate(self, this_camera: OpenCVCamera):- annotated_image = self.lens_distortion_calibrator.process_incoming_frame(this_camera.latest_frame)- # add something something cameras 6DoF calibration stuff here later-+ annotated_image = self._process_incoming_frame(this_camera.latest_frame)+return annotated_image++ def _process_incoming_frame(self, raw_frame_payload: FramePayload):++ charuco_frame_payload = self._charuco_board_detector.detect_charuco_board(raw_frame_payload)++ charuco_view_data = charuco_frame_payload.charuco_view_data++ if self._current_calibration is None:+ self._current_calibration = CameraCalibrationData(charuco_view_data.image_width,+ charuco_view_data.image_height)++ if not charuco_view_data.full_board_found and self._current_calibration is None:+ return charuco_frame_payload.annotated_image, self._current_calibration+++ if charuco_view_data.full_board_found:+ self._calibrate_camera(charuco_view_data) # this is where the magic happens+++ if charuco_view_data.some_charuco_corners_found:+ annotate_image_with_charuco_data(+ raw_frame_payload.image, # this image will have stuff drawn on top of it inside this function+ charuco_view_data,+ self._charuco_board_detector.number_of_charuco_corners+ )++ undistorted_image = cv2.undistort(raw_frame_payload.image,+ self._current_calibration.camera_matrix,+ self._current_calibration.lens_distortion_coefficients,+ )++ return undistorted_image, self._current_calibration++ def _calibrate_camera(self, charuco_view: CharucoViewData):+ """+ adapted from - https://mecaruco2.readthedocs.io/en/latest/notebooks_rst/Aruco/sandbox/ludovic/aruco_calibration_rotation.html++ Calibrates the camera using the charuco data++ helpful resources -+ https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html <-opencv's 3d camera calibration docs+ https://www.adamsmith.haus/python/docs/cv2.aruco.calibrateCameraCharucoExtended+ https://www.mathworks.com/help/vision/ug/camera-calibration.html+ https://docs.opencv.org/3.4/d9/db7/tutorial_py_table_of_contents_calib3d.html+ https://calib.io/blogs/knowledge-base/camera-models <-a good one+ """+ print("CAMERA CALIBRATION")++ image_width = charuco_view.image_width+ image_height = charuco_view.image_height+++ # flag definitions -> https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html#ga3207604e4b1a1758aa66acb6ed5aa65d+ flags = (cv2.CALIB_USE_INTRINSIC_GUESS ++ cv2.CALIB_ZERO_TANGENT_DIST ++ cv2.cv2.CALIB_FIX_ASPECT_RATIO ++ cv2.CALIB_FIX_PRINCIPAL_POINT ++ # cv2.CALIB_FIX_FOCAL_LENGTH+ cv2.CALIB_RATIONAL_MODEL+ # cv2.CALIB_THIN_PRISM_MODEL+ # cv2.CALIB_TILTED_MODEL+ )++ # https://docs.opencv.org/4.x/d9/d6a/group__aruco.html#gacf03e5afb0bc516b73028cf209984a06+ (this_reprojection_error,+ this_camera_matrix,+ these_lens_distortion_coefficients,+ these_rotation_vectors_of_the_board,+ these_translation_vectors_of_the_board,+ this_lens_distortion_std_dev,+ this_camera_location_std_dev,+ these_reprojection_error_per_view+ ) = cv2.aruco.calibrateCameraCharucoExtended(+ charucoCorners=[charuco_view.charuco_corners],+ charucoIds=[charuco_view.charuco_ids],+ board=self._charuco_board_detector.charuco_board,+ imageSize=(image_width, image_height),+ cameraMatrix=self._current_calibration.camera_matrix,+ distCoeffs=self._current_calibration.lens_distortion_coefficients,+ flags=flags,+ criteria=(cv2.TERM_CRITERIA_EPS & cv2.TERM_CRITERIA_COUNT, 10000, 1e-9))++ if this_reprojection_error < self._current_calibration.reprojection_error:+ self._current_calibration = CameraCalibrationData(+ reprojection_error=this_reprojection_error,+ camera_matrix=this_camera_matrix,+ lens_distortion_coefficients=these_lens_distortion_coefficients,+ image_width=image_width,+ image_height=image_height,+ rotation_vectors_of_the_board=these_rotation_vectors_of_the_board,+ translation_vectors_of_the_board=these_translation_vectors_of_the_board,+ lens_distortion_std_dev=this_lens_distortion_std_dev,+ camera_location_std_dev=this_camera_location_std_dev)diff --git a/src/core_processor/camera_calibration/charuco_board_detection/board_detector.py b/src/core_processor/camera_calibration/charuco_board_detection/charuco_board_detector.pysimilarity index 99%rename from src/core_processor/camera_calibration/charuco_board_detection/board_detector.pyrename to src/core_processor/camera_calibration/charuco_board_detection/charuco_board_detector.pyindex 7a5ae245..d710b270 100644--- a/src/core_processor/camera_calibration/charuco_board_detection/board_detector.py+++ b/src/core_processor/camera_calibration/charuco_board_detection/charuco_board_detector.py@@ -15,7 +15,7 @@ from src.core_processor.camera_calibration.charuco_board_detection.charuco_imagelogger = logging.getLogger(__name__)-class BoardDetector:+class CharucoBoardDetector:def __init__(self):charuco_board_object = CharucoBoard()self.charuco_board = charuco_board_object.charuco_boarddiff --git a/src/core_processor/camera_calibration/charuco_board_detection/dataclasses/charuco_board_definition.py b/src/core_processor/camera_calibration/charuco_board_detection/dataclasses/charuco_board_definition.pyindex 5c3fe9ee..08faa8c4 100644--- a/src/core_processor/camera_calibration/charuco_board_detection/dataclasses/charuco_board_definition.py+++ b/src/core_processor/camera_calibration/charuco_board_detection/dataclasses/charuco_board_definition.py@@ -2,6 +2,7 @@ from dataclasses import dataclassfrom typing import Dictimport cv2+import numpy as np@dataclass@@ -21,6 +22,7 @@ class CharucoBoard:self._number_of_charuco_corners = (self._number_of_squares_width - 1) * (self._number_of_squares_height - 1)+@propertydef charuco_board(self):return self._charuco_board@@ -32,3 +34,4 @@ class CharucoBoard:@propertydef aruco_marker_dict(self):return self._aruco_marker_dict+diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.pyindex b18498b9..5b4cbd46 100644--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py@@ -5,6 +5,7 @@ from typing import Dictimport cv2import numpy as npimport pyqtgraph as pg+import pyqtgraph.opengl as glfrom pyqtgraph.dockarea.Dock import Dockfrom pyqtgraph.dockarea.DockArea import DockAreafrom pyqtgraph.Qt import QtWidgets@@ -239,8 +240,8 @@ class QTVisualizerAndGui:passdef _setup_3d_viewport(self):- self.opengl_3d_plot_widget = pg.opengl.GLViewWidget()- self.opengl_grid_item = pg.opengl.GLGridItem()+ self.opengl_3d_plot_widget = gl.GLViewWidget()+ self.opengl_grid_item = gl.GLGridItem()self.opengl_3d_plot_widget.addItem(self.opengl_grid_item)# create XYZ axes@@ -264,7 +265,7 @@ class QTVisualizerAndGui:self.opengl_3d_plot_widget.addItem(self.origin_y_axis_gl_lineplot_item)self.opengl_3d_plot_widget.addItem(self.origin_z_axis_gl_lineplot_item)- self.opengl_charuco_scatter_item = pg.opengl.GLScatterPlotItem(pos=(0, 0, 0), color=(1, 0, 1), size=1,+ self.opengl_charuco_scatter_item = gl.GLScatterPlotItem(pos=(0, 0, 0), color=(1, 0, 1), size=1,pxMode=False)self.opengl_3d_plot_dock = Dock("3d View Port")