Free Motion Capture for Everyone πβ¨
TypeScript
ready to add camera calib code to `_calibration_work`
Jonb391b4f69422
diff --git a/requirements.txt b/requirements.txtindex 4f49816f..4edbf837 100644--- a/requirements.txt+++ b/requirements.txt@@ -2,30 +2,29 @@ aniposelib==0.4.3h5py>=3.1.0imutils~=0.5.4matplotlib~=3.1.3-mediapipe==0.8.8-opencv-contrib-python==3.4.14.51+# mediapipe==0.8.8+opencv-contrib-python==4.5.5.62+pandas~=1.2.5rich~=10.4.0tqdm~=4.61.1ruamel.yaml>=0.15.0-numpy<=1.19+numpy>=1.19typed-ast<1.5,>=1.4.0moviepy ~=1.0.3ipython-# pydantic==1.9.0-# uvicorn==0.16.0-# fastapi==0.71.0-# gunicorn==20.1.0-# requests==2.27.1-# pyudev==0.22.0-# python-multipart==0.0.5-# websockets==10.1-# sqlalchemy==1.4.31-# singleton-decorator==1.0.0-# orjson==3.6.7-# aiomultiprocess==0.9.0+pydantic==1.9.0+uvicorn==0.16.0+fastapi==0.71.0+gunicorn==20.1.0+requests==2.27.1+pyudev==0.22.0+python-multipart==0.0.5+websockets==10.1+sqlalchemy==1.4.31+singleton-decorator==1.0.0+orjson==3.6.7+aiomultiprocess==0.9.0# faster-fifo==1.3.0#viztracer==0.14.5-# pre-commit==2.17.0-# taskipy==1.10.1-# pre-commit==2.17.0+pre-commit==2.17.0# Requirements for core-processordiff --git a/src/cameras/capture/frame_payload.py b/src/cameras/capture/frame_payload.pyindex a38ad793..a94471e3 100644--- a/src/cameras/capture/frame_payload.py+++ b/src/cameras/capture/frame_payload.py@@ -7,3 +7,5 @@ class FramePayload(NamedTuple):success: bool = Falseimage: np.ndarray = Nonetimestamp: Union[int, float] = None++diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.pyindex a7a8fd0d..b77abac2 100644--- a/src/cameras/capture/opencv_camera/opencv_camera.py+++ b/src/cameras/capture/opencv_camera/opencv_camera.py@@ -76,7 +76,7 @@ class OpenCVCamera:self._opencv_video_capture_object = cv2.VideoCapture(self._config.webcam_id, cap_backend)- self._apply_configuration()+ # self._apply_configuration()success, image = self._opencv_video_capture_object.read()if not success:diff --git a/src/cameras/detection/cam_detection.py b/src/cameras/detection/cam_detection.pyindex 3494c2c1..d51fdad0 100644--- a/src/cameras/detection/cam_detection.py+++ b/src/cameras/detection/cam_detection.py@@ -5,7 +5,7 @@ import cv2from src.cameras.detection.models import FoundCamerasResponse, RawCamera-CAM_CHECK_NUM = 5+CAM_CHECK_NUM = 20logger = logging.getLogger(__name__)diff --git a/src/core_processor/board_detection/base_pose_estimation.py b/src/core_processor/board_detection/base_pose_estimation.pyindex ea9063b4..d0a43174 100644--- a/src/core_processor/board_detection/base_pose_estimation.py+++ b/src/core_processor/board_detection/base_pose_estimation.py@@ -1,9 +1,20 @@+from dataclasses import dataclass+from typing import List, Optional+import cv2+import numpy as npfrom src.core_processor.board_detection.charuco_constants import aruco_dict, board+@dataclass+class CharucoData:+ charuco_corners: Optional[np.ndarray]+ charuco_ids: Optional[np.ndarray]+ aruco_square_corners: Optional[np.ndarray]+ aruco_square_ids: Optional[np.ndarray]+-def detect_charuco_board(image):+def detect_charuco_board(image) -> CharucoData:"""Charuco base pose estimation.more-or-less copied from - https://mecaruco2.readthedocs.io/en/latest/notebooks_rst/Aruco@@ -34,4 +45,7 @@ def detect_charuco_board(image):charuco_corners = res2[1]charuco_ids = res2[2]- return charuco_corners, charuco_ids, aruco_square_corners, aruco_square_ids+ return CharucoData(charuco_corners=charuco_corners,+ charuco_ids=charuco_ids,+ aruco_square_corners=aruco_square_corners,+ aruco_square_ids=aruco_square_ids)diff --git a/src/core_processor/board_detection/board_detection.py b/src/core_processor/board_detection/board_detection.pyindex 36a91f28..67c20ffd 100644--- a/src/core_processor/board_detection/board_detection.py+++ b/src/core_processor/board_detection/board_detection.py@@ -1,6 +1,7 @@import loggingimport timeimport traceback+from dataclasses import dataclassfrom pathlib import Pathfrom types import FunctionTypefrom typing import Any, Callable, Optional@@ -11,7 +12,7 @@ import numpy as npfrom src.cameras.capture.frame_payload import FramePayloadfrom src.cameras.multicam_manager.cv_camera_manager import CVCameraManagerfrom src.cameras.persistence.video_writer.video_writer import SaveOptions-from src.core_processor.board_detection.base_pose_estimation import detect_charuco_board+from src.core_processor.board_detection.base_pose_estimation import detect_charuco_board, CharucoDatafrom src.core_processor.board_detection.charuco_image_annotator import (annotate_image_with_charuco_data,)@@ -21,9 +22,15 @@ from src.core_processor.utils.image_fps_writer import write_fps_to_imagelogger = logging.getLogger(__name__)+@dataclass+class CharucoFramePayload:+ raw_frame_payload: FramePayload+ annotated_frame_payload: FramePayload+ charuco_data: CharucoDataclass BoardDetection:def __init__(self, cam_manager: CVCameraManager = CVCameraManager()):+ print('pre-henlo')self._cam_manager = cam_managerasync def process_by_cam_id(self, webcam_id: str, cb):@@ -38,14 +45,10 @@ class BoardDetection:while True:if not cv_cam.is_capturing_frames:return- response = self._process_single_cam_frame(cv_cam)- if cb and response.image is not None:- writer.write(- FramePayload(- image=response.image, timestamp=response.timestamp- )- )- await cb(response.image)+ charuco_frame_payload = self._detect_charuco_board_in_image(cv_cam)+ if cb and charuco_frame_payload.annotated_frame_payload.image is not None:+ writer.write(charuco_frame_payload.annotated_frame_payload)+ await cb(charuco_frame_payload.annotated_frame_payload.image)except Exception as e:logger.error("Printing traceback")traceback.print_exc()@@ -65,7 +68,7 @@ class BoardDetection:writer.save(options)def process(- self, show_window=True, post_processed_frame_cb=Callable[[FramePayload], None]+ self, post_processed_frame_cb:Callable[[str, CharucoFramePayload], None]=None, show_window=True, save_video=True,):"""Opens Camera using OpenCV and begins image processing for charuco board@@ -82,19 +85,23 @@ class BoardDetection:for response in session_obj:cv_cam = response.cv_camwriter = response.writer- payload = self._process_single_cam_frame(cv_cam)+ charuco_frame_payload = self._detect_charuco_board_in_image(cv_cam)current_webcam_id = cv_cam.webcam_id_as_str- if payload is not None:- writer.write(payload)- if post_processed_frame_cb:- post_processed_frame_cb(payload)+ if not charuco_frame_payload:+ continue++ if save_video:+ writer.write(charuco_frame_payload.annotated_frame_payload)++ if post_processed_frame_cb:+ post_processed_frame_cb(current_webcam_id, charuco_frame_payload)- fps_manager.increment_frame_processed_for(current_webcam_id)- if show_window:- should_continue = show_cam_window(- current_webcam_id, payload.image, fps_manager- )+ fps_manager.increment_frame_processed_for(current_webcam_id)+ if show_window:+ should_continue = show_cam_window(+ current_webcam_id, charuco_frame_payload.annotated_frame_payload.image, fps_manager+ )except:logger.error("Printing traceback")traceback.print_exc()@@ -117,28 +124,32 @@ class BoardDetection:cv2.destroyWindow(cv_cam.webcam_id_as_str)cv2.waitKey(1)- def _process_single_cam_frame(self, cv_cam):- success, frame, timestamp = cv_cam.latest_frame- if not success:+ def _detect_charuco_board_in_image(self, cv_cam):+ raw_frame_payload = cv_cam.latest_frame++ if not raw_frame_payload.success:# logger.error("CV2 failed to grab a frame")return- if frame is None:+ if raw_frame_payload.image is None:logger.error("Frame is empty")return- (- charuco_corners,- charuco_ids,- aruco_square_corners,- aruco_square_ids,- ) = detect_charuco_board(frame)- # TODO - Pull out timestamps per frame and calculate fps to display on image+ charuco_data = detect_charuco_board(raw_frame_payload.image)++ annotated_image = raw_frame_payload.image.copy()+success_bool = annotate_image_with_charuco_data(- frame, cv_cam.webcam_id_as_str, charuco_corners, charuco_ids+ annotated_image, cv_cam.webcam_id_as_str, charuco_data+ )+ cv2.polylines(annotated_image, np.int32([charuco_data.charuco_corners]), True, (0, 100, 255), 2)++ return CharucoFramePayload(+ annotated_frame_payload=FramePayload(image=annotated_image, timestamp=time.time_ns()),+ raw_frame_payload=raw_frame_payload,+ charuco_data=charuco_data)- cv2.polylines(frame, np.int32([charuco_corners]), True, (0, 100, 255), 2)- return FramePayload(image=frame, timestamp=time.time_ns())if __name__ == "__main__":## Jon, Run me to easily start a charuco board detect run+ print('starting main')BoardDetection().process()diff --git a/src/core_processor/board_detection/charuco_image_annotator.py b/src/core_processor/board_detection/charuco_image_annotator.pyindex 116d763d..ed183c7b 100644--- a/src/core_processor/board_detection/charuco_image_annotator.py+++ b/src/core_processor/board_detection/charuco_image_annotator.py@@ -22,15 +22,18 @@ def _board_text(def annotate_image_with_charuco_data(- image, webcam_id: str, charuco_corners, charuco_ids+ image, webcam_id: str, charuco_data) -> bool:full_charuco_detected_on_this_frame = False- if len(charuco_ids) == num_charuco_corners:+ if len(charuco_data.charuco_ids) == num_charuco_corners:full_charuco_detected_on_this_frame = Trueimage_w_markers = cv2.aruco.drawDetectedCornersCharuco(- image, np.array(charuco_corners), np.array(charuco_ids), (0, 255, 125, 255)+ image,+ np.array(charuco_data.charuco_corners),+ np.array(charuco_data.charuco_ids),+ (0, 255, 125, 255)) # yellow? I# think cv2 uses BGR instead of RGB?@@ -39,7 +42,7 @@ def annotate_image_with_charuco_data(text_to_write_on_this_camera = ""current_cam_corner_count_str = _board_text(this_cam_name,- charuco_ids,+ charuco_data.charuco_ids,num_charuco_corners,full_charuco_detected_on_this_frame,)@@ -76,7 +79,7 @@ def annotate_image_with_charuco_data(if full_charuco_detected_on_this_frame:cv2.polylines(- image_w_markers, np.int32([charuco_corners]), False, (0, 100, 255), 2+ image_w_markers, np.int32([charuco_data.charuco_corners]), False, (0, 100, 255), 2)# for these_corners in charuco_points_from_previous_frames:# if len(these_corners)>0:diff --git a/src/core_processor/camera_calibration/camera_calibration.py b/src/core_processor/camera_calibration/camera_calibration.pyindex 14004ba0..06952714 100644--- a/src/core_processor/camera_calibration/camera_calibration.py+++ b/src/core_processor/camera_calibration/camera_calibration.py@@ -1,5 +1,5 @@from src.cameras.capture.frame_payload import FramePayload-from src.core_processor.board_detection.board_detection import BoardDetection+from src.core_processor.board_detection.board_detection import BoardDetection, CharucoFramePayloadclass CameraCalibration:@@ -9,14 +9,16 @@ class CameraCalibration:def perform_calibration(self):self._bd.process(show_window=True,+ save_video=True,post_processed_frame_cb=self._calibration_work,)- def _calibration_work(self, frame: FramePayload):- print("found frame", frame.timestamp)+ def _calibration_work(self, webcam_id: str, charuco_frame_payload: CharucoFramePayload):+ print(f"found frame from camera {webcam_id} at timestamp {charuco_frame_payload.annotated_frame_payload.timestamp/1e9}")if __name__ == "__main__":+ print('start main')a = BoardDetection()b = CameraCalibration(a)b.perform_calibration()