Free Motion Capture for Everyone πβ¨
TypeScript
proof-of-concept reprojection error visualizer
Jon4ff6bac9ecd4
diff --git a/requirements.txt b/requirements.txtindex c489e10d..91a54d89 100644--- a/requirements.txt+++ b/requirements.txt@@ -28,6 +28,6 @@ aiomultiprocess==0.9.0#viztracer==0.14.5pre-commit==2.17.0# Requirements for core-processor-#pyqtgraph==0.12.4-#pyqt5==5.15.6-#pyopengl==3.1.6\ No newline at end of file+pyqtgraph==0.12.4+pyqt5==5.15.6+pyopengl==3.1.6\ No newline at end of filediff --git a/src/core_processor/camera_calibration/calibration_diagnostics_visualizer.py b/src/core_processor/camera_calibration/calibration_diagnostics_visualizer.pynew file mode 100644index 00000000..022d3be7--- /dev/null+++ b/src/core_processor/camera_calibration/calibration_diagnostics_visualizer.py@@ -0,0 +1,30 @@+import numpy as np+import pyqtgraph as pg+++class CalibrationDiagnosticsVisualizer:+ def __init__(self):+ self.graphics_layout_window = pg.GraphicsLayoutWidget(show=True)+ self.graphics_layout_window.setWindowTitle('pyqtgraph example: Scrolling Plots')++ self.reprojection_error_all = np.empty(0)++ self.reprojection_error_subplot = self.graphics_layout_window.addPlot()+ self.reprojection_error_line = self.reprojection_error_subplot.plot(self.reprojection_error_all)++ self.subplot_1 = self.graphics_layout_window.addPlot()+ self.data1 = np.random.normal(size=300)+ self.curve2 = self.subplot_1.plot(self.data1)+ self.ptr1 = 0++ def update(self, reprojection_error_current):+ # update reprojection error subplot+ self.reprojection_error_all = np.append(self.reprojection_error_all, reprojection_error_current)+ self.reprojection_error_line.setData(self.reprojection_error_all)++ # update other subplot+ self.curve2.setData(self.data1)+ self.curve2.setPos(self.ptr1, 0)++ def close(self):+ self.graphics_layout_window.close()diff --git a/src/core_processor/camera_calibration/camera_calibration.py b/src/core_processor/camera_calibration/camera_calibration.pyindex bf1b2b88..378b1601 100644--- a/src/core_processor/camera_calibration/camera_calibration.py+++ b/src/core_processor/camera_calibration/camera_calibration.py@@ -1,35 +1,38 @@from dataclasses import dataclassimport timeimport cv2+import pyqtgraph as pgimport numpy as npfrom rich import printfrom src.core_processor.board_detection.board_detection import BoardDetection, CharucoFramePayloadfrom src.core_processor.board_detection.charuco_constants import charuco_board_object--number_of_distortion_coefficients = 5 # may be 4, 5, 8, or 12, not sure of the trade-off there+from src.core_processor.camera_calibration.calibration_diagnostics_visualizer import CalibrationDiagnosticsVisualizer@dataclassclass SingleCameraCalibrationData:reprojection_error: float = 1e9camera_matrix: np.ndarray = None- lens_distortion_coefficients: np.ndarray = np.zeros((number_of_distortion_coefficients, 1))+ lens_distortion_coefficients: np.ndarray = Nonerotation_vectors_of_the_board: np.ndarray = Nonetranslation_vectors_of_the_board: np.ndarray = Nonelens_distortion_std_dev: float = Nonecamera_location_std_dev: float = None+class CameraCalibration:- def __init__(self, board_detection: BoardDetection):+ def __init__(self, board_detection: BoardDetection, visualize_calibration_disagnostics=True, ):self._bd = board_detection-self._current_calibration = SingleCameraCalibrationData()self._start_time = time.time()self._all_charuco_corners_list = []self._all_charuco_ids_list = []+ if visualize_calibration_disagnostics:+ self.calibration_diagnostics_visualizer = CalibrationDiagnosticsVisualizer()+def perform_calibration(self):self._bd.process(show_window=True,@@ -49,7 +52,7 @@ class CameraCalibration:self._all_charuco_corners_list.append(charuco_frame_payload.charuco_data.charuco_corners)self._all_charuco_ids_list.append(charuco_frame_payload.charuco_data.charuco_ids)- self.estimate_lens_distortion(image_width, image_height,)+ self.estimate_lens_distortion(image_width, image_height, )# create new camera matrix that will show full undistored image (with black pixels in spots with no data (only for debugging)new_camera_matrix, valid_ROI_lbwh = cv2.getOptimalNewCameraMatrix(self._current_calibration.camera_matrix,@@ -66,6 +69,8 @@ class CameraCalibration:None,new_camera_matrix)+ self.calibration_diagnostics_visualizer.update(self._current_calibration.reprojection_error)+print(f"found frame from camera {webcam_id} at timestamp {elapsed_time:.3f} - reprojection error: {self._current_calibration.reprojection_error:.3f} - num charuco views:{len(self._all_charuco_corners_list)}")# print(f'lens distortion coefficients - {np.array_str(np.squeeze(self._current_calibration.lens_distortion_coefficients), precision=2)}')@@ -74,14 +79,14 @@ class CameraCalibration:return undistorted_imagedef estimate_lens_distortion(self, image_width, image_height,- max_iterations=10, max_charuco_board_views=5):+ max_iterations=5, max_charuco_board_views=10):"""adapted from - https://mecaruco2.readthedocs.io/en/latest/notebooks_rst/Aruco/sandbox/ludovic/aruco_calibration_rotation.htmlCalibrates the camera using the charuco data using a RANSAC-like method - randomly selects board views and saves the combo that minimized reprojection errorhelpful resources -- https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html#ga3207604e4b1a1758aa66acb6ed5aa65d <-opencv's 3d camera calibration docs+ https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html <-opencv's 3d camera calibration docshttps://www.adamsmith.haus/python/docs/cv2.aruco.calibrateCameraCharucoExtendedhttps://www.mathworks.com/help/vision/ug/camera-calibration.htmlhttps://docs.opencv.org/3.4/d9/db7/tutorial_py_table_of_contents_calib3d.html@@ -90,28 +95,38 @@ class CameraCalibration:print("CAMERA CALIBRATION")initial_camera_matrix = np.array([[float(image_width), 0., image_width / 2.],- [0., float(image_width), image_height / 2.],- [0., 0., 1.]])+ [0., float(image_width), image_height / 2.],+ [0., 0., 1.]])+ number_of_distortion_coefficients = 4 # may be 4, 5, 8, or 12, not sure of the trade-off there but low numbers seem more stable+ initial_lens_distortion_coefficients = np.zeros((number_of_distortion_coefficients, 1))num_charuco_views = len(self._all_charuco_ids_list)# flag definitions -> https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html#ga3207604e4b1a1758aa66acb6ed5aa65d- flags = (cv2.CALIB_FIX_ASPECT_RATIO + cv2.CALIB_FIX_PRINCIPAL_POINT + cv2.CALIB_RATIONAL_MODEL)#+ cv2.CALIB_THIN_PRISM_MODEL)# + cv2.CALIB_TILTED_MODEL )+ flags = (cv2.CALIB_USE_INTRINSIC_GUESS ++ # cv2.CALIB_ZERO_TANGENT_DIST ++ cv2.cv2.CALIB_FIX_ASPECT_RATIO ++ cv2.CALIB_FIX_PRINCIPAL_POINT ++ cv2.CALIB_RATIONAL_MODEL+ # cv2.CALIB_THIN_PRISM_MODEL+ # cv2.CALIB_TILTED_MODEL+ )# flags = (cv2.CALIB_RATIONAL_MODEL)for iter in range(max_iterations):- if num_charuco_views<2:+ if num_charuco_views < 2:these_charuco_corners = self._all_charuco_corners_listthese_charuco_ids = self._all_charuco_ids_listelse:this_random_sample_of_charuco_views = np.random.choice(num_charuco_views,- size=min(num_charuco_views, max_charuco_board_views),- replace=False)- these_charuco_corners = [self._all_charuco_corners_list[cc] for cc in this_random_sample_of_charuco_views]+ size=min(num_charuco_views,+ max_charuco_board_views),+ replace=False)+ these_charuco_corners = [self._all_charuco_corners_list[cc] for cc in+ this_random_sample_of_charuco_views]these_charuco_ids = [self._all_charuco_ids_list[cc] for cc in this_random_sample_of_charuco_views]-# https://docs.opencv.org/4.x/d9/d6a/group__aruco.html#gacf03e5afb0bc516b73028cf209984a06(reprojection_error,camera_matrix,@@ -127,7 +142,7 @@ class CameraCalibration:board=charuco_board_object,imageSize=(image_width, image_height),cameraMatrix=initial_camera_matrix,- distCoeffs=np.zeros((number_of_distortion_coefficients, 1)),+ distCoeffs=initial_lens_distortion_coefficients,flags=flags,criteria=(cv2.TERM_CRITERIA_EPS & cv2.TERM_CRITERIA_COUNT, 10000, 1e-9))@@ -146,3 +161,4 @@ if __name__ == "__main__":a = BoardDetection()b = CameraCalibration(a)b.perform_calibration()+ b.calibration_diagnostics_visualizer.close()