Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

proof-of-concept reprojection error visualizer

Jon4ff6bac9ecd4
diff --git a/requirements.txt b/requirements.txt
index c489e10d..91a54d89 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -28,6 +28,6 @@ aiomultiprocess==0.9.0
#viztracer==0.14.5
pre-commit==2.17.0
# Requirements for core-processor
-#pyqtgraph==0.12.4
-#pyqt5==5.15.6
-#pyopengl==3.1.6
\ No newline at end of file
+pyqtgraph==0.12.4
+pyqt5==5.15.6
+pyopengl==3.1.6
\ No newline at end of file
diff --git a/src/core_processor/camera_calibration/calibration_diagnostics_visualizer.py b/src/core_processor/camera_calibration/calibration_diagnostics_visualizer.py
new file mode 100644
index 00000000..022d3be7
--- /dev/null
+++ b/src/core_processor/camera_calibration/calibration_diagnostics_visualizer.py
@@ -0,0 +1,30 @@
+import numpy as np
+import pyqtgraph as pg
+
+
+class CalibrationDiagnosticsVisualizer:
+ def __init__(self):
+ self.graphics_layout_window = pg.GraphicsLayoutWidget(show=True)
+ self.graphics_layout_window.setWindowTitle('pyqtgraph example: Scrolling Plots')
+
+ self.reprojection_error_all = np.empty(0)
+
+ self.reprojection_error_subplot = self.graphics_layout_window.addPlot()
+ self.reprojection_error_line = self.reprojection_error_subplot.plot(self.reprojection_error_all)
+
+ self.subplot_1 = self.graphics_layout_window.addPlot()
+ self.data1 = np.random.normal(size=300)
+ self.curve2 = self.subplot_1.plot(self.data1)
+ self.ptr1 = 0
+
+ def update(self, reprojection_error_current):
+ # update reprojection error subplot
+ self.reprojection_error_all = np.append(self.reprojection_error_all, reprojection_error_current)
+ self.reprojection_error_line.setData(self.reprojection_error_all)
+
+ # update other subplot
+ self.curve2.setData(self.data1)
+ self.curve2.setPos(self.ptr1, 0)
+
+ def close(self):
+ self.graphics_layout_window.close()
diff --git a/src/core_processor/camera_calibration/camera_calibration.py b/src/core_processor/camera_calibration/camera_calibration.py
index bf1b2b88..378b1601 100644
--- a/src/core_processor/camera_calibration/camera_calibration.py
+++ b/src/core_processor/camera_calibration/camera_calibration.py
@@ -1,35 +1,38 @@
from dataclasses import dataclass
import time
import cv2
+import pyqtgraph as pg
import numpy as np
from rich import print
from src.core_processor.board_detection.board_detection import BoardDetection, CharucoFramePayload
from src.core_processor.board_detection.charuco_constants import charuco_board_object
-
-number_of_distortion_coefficients = 5 # may be 4, 5, 8, or 12, not sure of the trade-off there
+from src.core_processor.camera_calibration.calibration_diagnostics_visualizer import CalibrationDiagnosticsVisualizer
@dataclass
class SingleCameraCalibrationData:
reprojection_error: float = 1e9
camera_matrix: np.ndarray = None
- lens_distortion_coefficients: np.ndarray = np.zeros((number_of_distortion_coefficients, 1))
+ lens_distortion_coefficients: np.ndarray = None
rotation_vectors_of_the_board: np.ndarray = None
translation_vectors_of_the_board: np.ndarray = None
lens_distortion_std_dev: float = None
camera_location_std_dev: float = None
+
class CameraCalibration:
- def __init__(self, board_detection: BoardDetection):
+ def __init__(self, board_detection: BoardDetection, visualize_calibration_disagnostics=True, ):
self._bd = board_detection
-
self._current_calibration = SingleCameraCalibrationData()
self._start_time = time.time()
self._all_charuco_corners_list = []
self._all_charuco_ids_list = []
+ if visualize_calibration_disagnostics:
+ self.calibration_diagnostics_visualizer = CalibrationDiagnosticsVisualizer()
+
def perform_calibration(self):
self._bd.process(
show_window=True,
@@ -49,7 +52,7 @@ class CameraCalibration:
self._all_charuco_corners_list.append(charuco_frame_payload.charuco_data.charuco_corners)
self._all_charuco_ids_list.append(charuco_frame_payload.charuco_data.charuco_ids)
- self.estimate_lens_distortion(image_width, image_height,)
+ self.estimate_lens_distortion(image_width, image_height, )
# create new camera matrix that will show full undistored image (with black pixels in spots with no data (only for debugging)
new_camera_matrix, valid_ROI_lbwh = cv2.getOptimalNewCameraMatrix(self._current_calibration.camera_matrix,
@@ -66,6 +69,8 @@ class CameraCalibration:
None,
new_camera_matrix)
+ self.calibration_diagnostics_visualizer.update(self._current_calibration.reprojection_error)
+
print(
f"found frame from camera {webcam_id} at timestamp {elapsed_time:.3f} - reprojection error: {self._current_calibration.reprojection_error:.3f} - num charuco views:{len(self._all_charuco_corners_list)}")
# print(f'lens distortion coefficients - {np.array_str(np.squeeze(self._current_calibration.lens_distortion_coefficients), precision=2)}')
@@ -74,14 +79,14 @@ class CameraCalibration:
return undistorted_image
def estimate_lens_distortion(self, image_width, image_height,
- max_iterations=10, max_charuco_board_views=5):
+ max_iterations=5, max_charuco_board_views=10):
"""
adapted from - https://mecaruco2.readthedocs.io/en/latest/notebooks_rst/Aruco/sandbox/ludovic/aruco_calibration_rotation.html
Calibrates the camera using the charuco data using a RANSAC-like method - randomly selects board views and saves the combo that minimized reprojection error
helpful resources -
- https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html#ga3207604e4b1a1758aa66acb6ed5aa65d <-opencv's 3d camera calibration docs
+ https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html <-opencv's 3d camera calibration docs
https://www.adamsmith.haus/python/docs/cv2.aruco.calibrateCameraCharucoExtended
https://www.mathworks.com/help/vision/ug/camera-calibration.html
https://docs.opencv.org/3.4/d9/db7/tutorial_py_table_of_contents_calib3d.html
@@ -90,28 +95,38 @@ class CameraCalibration:
print("CAMERA CALIBRATION")
initial_camera_matrix = np.array([[float(image_width), 0., image_width / 2.],
- [0., float(image_width), image_height / 2.],
- [0., 0., 1.]])
+ [0., float(image_width), image_height / 2.],
+ [0., 0., 1.]])
+ number_of_distortion_coefficients = 4 # may be 4, 5, 8, or 12, not sure of the trade-off there but low numbers seem more stable
+ initial_lens_distortion_coefficients = np.zeros((number_of_distortion_coefficients, 1))
num_charuco_views = len(self._all_charuco_ids_list)
# flag definitions -> https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html#ga3207604e4b1a1758aa66acb6ed5aa65d
- flags = (cv2.CALIB_FIX_ASPECT_RATIO + cv2.CALIB_FIX_PRINCIPAL_POINT + cv2.CALIB_RATIONAL_MODEL)#+ cv2.CALIB_THIN_PRISM_MODEL)# + cv2.CALIB_TILTED_MODEL )
+ flags = (cv2.CALIB_USE_INTRINSIC_GUESS +
+ # cv2.CALIB_ZERO_TANGENT_DIST +
+ cv2.cv2.CALIB_FIX_ASPECT_RATIO +
+ cv2.CALIB_FIX_PRINCIPAL_POINT +
+ cv2.CALIB_RATIONAL_MODEL
+ # cv2.CALIB_THIN_PRISM_MODEL
+ # cv2.CALIB_TILTED_MODEL
+ )
# flags = (cv2.CALIB_RATIONAL_MODEL)
for iter in range(max_iterations):
- if num_charuco_views<2:
+ if num_charuco_views < 2:
these_charuco_corners = self._all_charuco_corners_list
these_charuco_ids = self._all_charuco_ids_list
else:
this_random_sample_of_charuco_views = np.random.choice(num_charuco_views,
- size=min(num_charuco_views, max_charuco_board_views),
- replace=False)
- these_charuco_corners = [self._all_charuco_corners_list[cc] for cc in this_random_sample_of_charuco_views]
+ size=min(num_charuco_views,
+ max_charuco_board_views),
+ replace=False)
+ these_charuco_corners = [self._all_charuco_corners_list[cc] for cc in
+ this_random_sample_of_charuco_views]
these_charuco_ids = [self._all_charuco_ids_list[cc] for cc in this_random_sample_of_charuco_views]
-
# https://docs.opencv.org/4.x/d9/d6a/group__aruco.html#gacf03e5afb0bc516b73028cf209984a06
(reprojection_error,
camera_matrix,
@@ -127,7 +142,7 @@ class CameraCalibration:
board=charuco_board_object,
imageSize=(image_width, image_height),
cameraMatrix=initial_camera_matrix,
- distCoeffs=np.zeros((number_of_distortion_coefficients, 1)),
+ distCoeffs=initial_lens_distortion_coefficients,
flags=flags,
criteria=(cv2.TERM_CRITERIA_EPS & cv2.TERM_CRITERIA_COUNT, 10000, 1e-9))
@@ -146,3 +161,4 @@ if __name__ == "__main__":
a = BoardDetection()
b = CameraCalibration(a)
b.perform_calibration()
+ b.calibration_diagnostics_visualizer.close()