Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

check in during twitch

Jone5398b9e86fb
diff --git a/requirements.txt b/requirements.txt
index d1095aea..3772c764 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,17 +1,17 @@
-aniposelib==0.4.3
-h5py>=3.1.0
-imutils~=0.5.4
-matplotlib~=3.5.1
-# mediapipe==0.8.8
-opencv-contrib-python==4.5.5.62
-pandas~=1.2.5
-rich~=10.4.0
-tqdm~=4.61.1
-ruamel.yaml>=0.15.0
-numpy>=1.19
-typed-ast<1.5,>=1.4.0
-moviepy ~=1.0.3
+#aniposelib==0.4.3
+#h5py>=3.1.0
+#imutils~=0.5.4
+#matplotlib~=3.5.1
+## mediapipe==0.8.8
+#opencv-contrib-python==4.5.5.62
+#pandas~=1.2.5
+#rich~=10.4.0
+#tqdm~=4.61.1
+#ruamel.yaml>=0.15.0
+#typed-ast<1.5,>=1.4.0
+#moviepy ~=1.0.3
ipython
+numpy>=1.19
pydantic==1.9.0
uvicorn==0.16.0
fastapi==0.71.0
diff --git a/src/api/services/board_detect_service.py b/src/api/services/board_detect_service.py
index e87a7aad..4f4aca3e 100644
--- a/src/api/services/board_detect_service.py
+++ b/src/api/services/board_detect_service.py
@@ -1,7 +1,7 @@
from typing import NamedTuple
from src.cameras.multicam_manager import CVCameraManager
-from src.core_processor.camera_calibration.charuco_board_detection.board_detector import BoardDetector
+from src.core_processor.camera_calibration.charuco_board_detection.charuco_board_detector import CharucoBoardDetector
class ImageResponse(NamedTuple):
@@ -14,7 +14,7 @@ class BoardDetectService:
self._cam_manager = cam_manager
async def run(self):
- return BoardDetector(self._cam_manager).process()
+ return CharucoBoardDetector(self._cam_manager).process()
async def run_detection_on_cam_id(self, webcam_id: str, cb=None):
- await BoardDetector(self._cam_manager).process_by_cam_id(webcam_id, cb)
+ await CharucoBoardDetector(self._cam_manager).process_by_cam_id(webcam_id, cb)
diff --git a/src/core_processor/camera_calibration/calibration_dataclasses.py b/src/core_processor/camera_calibration/calibration_dataclasses.py
index 0dae6848..85a19ea8 100644
--- a/src/core_processor/camera_calibration/calibration_dataclasses.py
+++ b/src/core_processor/camera_calibration/calibration_dataclasses.py
@@ -3,9 +3,8 @@ from dataclasses import dataclass
import numpy as np
-
@dataclass
-class LensDistortionCalibrationData:
+class CameraCalibrationData:
image_width: int
image_height: int
reprojection_error: float = 1e9
@@ -26,17 +25,4 @@ class LensDistortionCalibrationData:
if self.lens_distortion_coefficients is None:
self.lens_distortion_coefficients = np.zeros((self.number_of_lens_distortion_coefficients, 1))
-@dataclass
-class CameraCalibrationData:
- image_width: int
- image_height: int
- lens_distortion_calibration_data: LensDistortionCalibrationData = None
- camera_translation_relative_to_camera0: np.ndarray = np.zeros((3, 1))
- camera_rotation_relative_to_camera0: np.ndarray = np.zeros((3, 1))
-
- def __init__(self, image_width, image_height):
- self.image_width = image_width
- self.image_height = image_height
- self.lens_distortion_calibration_data = LensDistortionCalibrationData(self.image_width,
- self.image_height)
diff --git a/src/core_processor/camera_calibration/calibration_diagnostics_visualizer.py b/src/core_processor/camera_calibration/calibration_diagnostics_visualizer.py
index 362b7b69..6a965497 100644
--- a/src/core_processor/camera_calibration/calibration_diagnostics_visualizer.py
+++ b/src/core_processor/camera_calibration/calibration_diagnostics_visualizer.py
@@ -7,7 +7,7 @@ from pyqtgraph.Qt import QtWidgets
import cv2
-from src.core_processor.camera_calibration.calibration_dataclasses import LensDistortionCalibrationData
+from src.core_processor.camera_calibration.calibration_dataclasses import CameraCalibrationData
@@ -134,7 +134,7 @@ class CalibrationDiagnosticsVisualizer:
self.reprojection_error_all_line.setData(self.reprojection_error_all)
self.reprojection_error_current_best_line.setData(self.reprojection_error_current_best)
- def update_calibration_text_overlay(self, calibration_data: LensDistortionCalibrationData):
+ def update_calibration_text_overlay(self, calibration_data: CameraCalibrationData):
lens_distortion_as_str = [f"{d[0]:.3f}" for d in calibration_data.lens_distortion_coefficients]
calibration_text = f'reprojection error:{calibration_data.reprojection_error:.3f} \n {lens_distortion_as_str} \n {str(calibration_data.camera_matrix)}'
self.calibration_text_widget.setText(calibration_text)
diff --git a/src/core_processor/camera_calibration/camera_calibrator.py b/src/core_processor/camera_calibration/camera_calibrator.py
index 48d2e5cf..fe0146b7 100644
--- a/src/core_processor/camera_calibration/camera_calibrator.py
+++ b/src/core_processor/camera_calibration/camera_calibrator.py
@@ -1,21 +1,121 @@
import logging
from dataclasses import dataclass
+import cv2
import numpy as np
+from src.cameras.capture.dataclasses.frame_payload import FramePayload
from src.cameras.capture.opencv_camera.opencv_camera import OpenCVCamera
-from src.core_processor.camera_calibration.lens_distortion_calibrator import LensDistortionCalibrator, \
- LensDistortionCalibrationData
+from src.core_processor.camera_calibration.calibration_dataclasses import CameraCalibrationData
+from src.core_processor.camera_calibration.charuco_board_detection.charuco_board_detector import CharucoBoardDetector
+from src.core_processor.camera_calibration.charuco_board_detection.charuco_image_annotator import \
+ annotate_image_with_charuco_data
+from src.core_processor.camera_calibration.charuco_board_detection.dataclasses.charuco_view_data import CharucoViewData
logger = logging.getLogger(__name__)
class CameraCalibrator:
- def __init__(self):
- self.lens_distortion_calibrator = LensDistortionCalibrator()
+ def __init__(self,
+ charuco_board_detector: CharucoBoardDetector = CharucoBoardDetector(),
+ ):
+ self._current_calibration = None
+ self._charuco_board_detector = charuco_board_detector
def calibrate(self, this_camera: OpenCVCamera):
- annotated_image = self.lens_distortion_calibrator.process_incoming_frame(this_camera.latest_frame)
- # add something something cameras 6DoF calibration stuff here later
-
+ annotated_image = self._process_incoming_frame(this_camera.latest_frame)
+
return annotated_image
+
+ def _process_incoming_frame(self, raw_frame_payload: FramePayload):
+
+ charuco_frame_payload = self._charuco_board_detector.detect_charuco_board(raw_frame_payload)
+
+ charuco_view_data = charuco_frame_payload.charuco_view_data
+
+ if self._current_calibration is None:
+ self._current_calibration = CameraCalibrationData(charuco_view_data.image_width,
+ charuco_view_data.image_height)
+
+ if not charuco_view_data.full_board_found and self._current_calibration is None:
+ return charuco_frame_payload.annotated_image, self._current_calibration
+
+
+ if charuco_view_data.full_board_found:
+ self._calibrate_camera(charuco_view_data) # this is where the magic happens
+
+
+ if charuco_view_data.some_charuco_corners_found:
+ annotate_image_with_charuco_data(
+ raw_frame_payload.image, # this image will have stuff drawn on top of it inside this function
+ charuco_view_data,
+ self._charuco_board_detector.number_of_charuco_corners
+ )
+
+ undistorted_image = cv2.undistort(raw_frame_payload.image,
+ self._current_calibration.camera_matrix,
+ self._current_calibration.lens_distortion_coefficients,
+ )
+
+ return undistorted_image, self._current_calibration
+
+ def _calibrate_camera(self, charuco_view: CharucoViewData):
+ """
+ adapted from - https://mecaruco2.readthedocs.io/en/latest/notebooks_rst/Aruco/sandbox/ludovic/aruco_calibration_rotation.html
+
+ Calibrates the camera using the charuco data
+
+ helpful resources -
+ https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html <-opencv's 3d camera calibration docs
+ https://www.adamsmith.haus/python/docs/cv2.aruco.calibrateCameraCharucoExtended
+ https://www.mathworks.com/help/vision/ug/camera-calibration.html
+ https://docs.opencv.org/3.4/d9/db7/tutorial_py_table_of_contents_calib3d.html
+ https://calib.io/blogs/knowledge-base/camera-models <-a good one
+ """
+ print("CAMERA CALIBRATION")
+
+ image_width = charuco_view.image_width
+ image_height = charuco_view.image_height
+
+
+ # flag definitions -> https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html#ga3207604e4b1a1758aa66acb6ed5aa65d
+ flags = (cv2.CALIB_USE_INTRINSIC_GUESS +
+ cv2.CALIB_ZERO_TANGENT_DIST +
+ cv2.cv2.CALIB_FIX_ASPECT_RATIO +
+ cv2.CALIB_FIX_PRINCIPAL_POINT +
+ # cv2.CALIB_FIX_FOCAL_LENGTH
+ cv2.CALIB_RATIONAL_MODEL
+ # cv2.CALIB_THIN_PRISM_MODEL
+ # cv2.CALIB_TILTED_MODEL
+ )
+
+ # https://docs.opencv.org/4.x/d9/d6a/group__aruco.html#gacf03e5afb0bc516b73028cf209984a06
+ (this_reprojection_error,
+ this_camera_matrix,
+ these_lens_distortion_coefficients,
+ these_rotation_vectors_of_the_board,
+ these_translation_vectors_of_the_board,
+ this_lens_distortion_std_dev,
+ this_camera_location_std_dev,
+ these_reprojection_error_per_view
+ ) = cv2.aruco.calibrateCameraCharucoExtended(
+ charucoCorners=[charuco_view.charuco_corners],
+ charucoIds=[charuco_view.charuco_ids],
+ board=self._charuco_board_detector.charuco_board,
+ imageSize=(image_width, image_height),
+ cameraMatrix=self._current_calibration.camera_matrix,
+ distCoeffs=self._current_calibration.lens_distortion_coefficients,
+ flags=flags,
+ criteria=(cv2.TERM_CRITERIA_EPS & cv2.TERM_CRITERIA_COUNT, 10000, 1e-9))
+
+ if this_reprojection_error < self._current_calibration.reprojection_error:
+ self._current_calibration = CameraCalibrationData(
+ reprojection_error=this_reprojection_error,
+ camera_matrix=this_camera_matrix,
+ lens_distortion_coefficients=these_lens_distortion_coefficients,
+ image_width=image_width,
+ image_height=image_height,
+ rotation_vectors_of_the_board=these_rotation_vectors_of_the_board,
+ translation_vectors_of_the_board=these_translation_vectors_of_the_board,
+ lens_distortion_std_dev=this_lens_distortion_std_dev,
+ camera_location_std_dev=this_camera_location_std_dev)
diff --git a/src/core_processor/camera_calibration/charuco_board_detection/board_detector.py b/src/core_processor/camera_calibration/charuco_board_detection/charuco_board_detector.py
similarity index 99%
rename from src/core_processor/camera_calibration/charuco_board_detection/board_detector.py
rename to src/core_processor/camera_calibration/charuco_board_detection/charuco_board_detector.py
index 7a5ae245..d710b270 100644
--- a/src/core_processor/camera_calibration/charuco_board_detection/board_detector.py
+++ b/src/core_processor/camera_calibration/charuco_board_detection/charuco_board_detector.py
@@ -15,7 +15,7 @@ from src.core_processor.camera_calibration.charuco_board_detection.charuco_image
logger = logging.getLogger(__name__)
-class BoardDetector:
+class CharucoBoardDetector:
def __init__(self):
charuco_board_object = CharucoBoard()
self.charuco_board = charuco_board_object.charuco_board
diff --git a/src/core_processor/camera_calibration/charuco_board_detection/dataclasses/charuco_board_definition.py b/src/core_processor/camera_calibration/charuco_board_detection/dataclasses/charuco_board_definition.py
index 5c3fe9ee..08faa8c4 100644
--- a/src/core_processor/camera_calibration/charuco_board_detection/dataclasses/charuco_board_definition.py
+++ b/src/core_processor/camera_calibration/charuco_board_detection/dataclasses/charuco_board_definition.py
@@ -2,6 +2,7 @@ from dataclasses import dataclass
from typing import Dict
import cv2
+import numpy as np
@dataclass
@@ -21,6 +22,7 @@ class CharucoBoard:
self._number_of_charuco_corners = (self._number_of_squares_width - 1) * (self._number_of_squares_height - 1)
+
@property
def charuco_board(self):
return self._charuco_board
@@ -32,3 +34,4 @@ class CharucoBoard:
@property
def aruco_marker_dict(self):
return self._aruco_marker_dict
+
diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
index b18498b9..5b4cbd46 100644
--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
@@ -5,6 +5,7 @@ from typing import Dict
import cv2
import numpy as np
import pyqtgraph as pg
+import pyqtgraph.opengl as gl
from pyqtgraph.dockarea.Dock import Dock
from pyqtgraph.dockarea.DockArea import DockArea
from pyqtgraph.Qt import QtWidgets
@@ -239,8 +240,8 @@ class QTVisualizerAndGui:
pass
def _setup_3d_viewport(self):
- self.opengl_3d_plot_widget = pg.opengl.GLViewWidget()
- self.opengl_grid_item = pg.opengl.GLGridItem()
+ self.opengl_3d_plot_widget = gl.GLViewWidget()
+ self.opengl_grid_item = gl.GLGridItem()
self.opengl_3d_plot_widget.addItem(self.opengl_grid_item)
# create XYZ axes
@@ -264,7 +265,7 @@ class QTVisualizerAndGui:
self.opengl_3d_plot_widget.addItem(self.origin_y_axis_gl_lineplot_item)
self.opengl_3d_plot_widget.addItem(self.origin_z_axis_gl_lineplot_item)
- self.opengl_charuco_scatter_item = pg.opengl.GLScatterPlotItem(pos=(0, 0, 0), color=(1, 0, 1), size=1,
+ self.opengl_charuco_scatter_item = gl.GLScatterPlotItem(pos=(0, 0, 0), color=(1, 0, 1), size=1,
pxMode=False)
self.opengl_3d_plot_dock = Dock("3d View Port")