Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

ready to add camera calib code to `_calibration_work`

Jonb391b4f69422
diff --git a/requirements.txt b/requirements.txt
index 4f49816f..4edbf837 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,30 +2,29 @@ aniposelib==0.4.3
h5py>=3.1.0
imutils~=0.5.4
matplotlib~=3.1.3
-mediapipe==0.8.8
-opencv-contrib-python==3.4.14.51
+# mediapipe==0.8.8
+opencv-contrib-python==4.5.5.62
+pandas~=1.2.5
rich~=10.4.0
tqdm~=4.61.1
ruamel.yaml>=0.15.0
-numpy<=1.19
+numpy>=1.19
typed-ast<1.5,>=1.4.0
moviepy ~=1.0.3
ipython
-# pydantic==1.9.0
-# uvicorn==0.16.0
-# fastapi==0.71.0
-# gunicorn==20.1.0
-# requests==2.27.1
-# pyudev==0.22.0
-# python-multipart==0.0.5
-# websockets==10.1
-# sqlalchemy==1.4.31
-# singleton-decorator==1.0.0
-# orjson==3.6.7
-# aiomultiprocess==0.9.0
+pydantic==1.9.0
+uvicorn==0.16.0
+fastapi==0.71.0
+gunicorn==20.1.0
+requests==2.27.1
+pyudev==0.22.0
+python-multipart==0.0.5
+websockets==10.1
+sqlalchemy==1.4.31
+singleton-decorator==1.0.0
+orjson==3.6.7
+aiomultiprocess==0.9.0
# faster-fifo==1.3.0
#viztracer==0.14.5
-# pre-commit==2.17.0
-# taskipy==1.10.1
-# pre-commit==2.17.0
+pre-commit==2.17.0
# Requirements for core-processor
diff --git a/src/cameras/capture/frame_payload.py b/src/cameras/capture/frame_payload.py
index a38ad793..a94471e3 100644
--- a/src/cameras/capture/frame_payload.py
+++ b/src/cameras/capture/frame_payload.py
@@ -7,3 +7,5 @@ class FramePayload(NamedTuple):
success: bool = False
image: np.ndarray = None
timestamp: Union[int, float] = None
+
+
diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.py
index a7a8fd0d..b77abac2 100644
--- a/src/cameras/capture/opencv_camera/opencv_camera.py
+++ b/src/cameras/capture/opencv_camera/opencv_camera.py
@@ -76,7 +76,7 @@ class OpenCVCamera:
self._opencv_video_capture_object = cv2.VideoCapture(
self._config.webcam_id, cap_backend
)
- self._apply_configuration()
+ # self._apply_configuration()
success, image = self._opencv_video_capture_object.read()
if not success:
diff --git a/src/cameras/detection/cam_detection.py b/src/cameras/detection/cam_detection.py
index 3494c2c1..d51fdad0 100644
--- a/src/cameras/detection/cam_detection.py
+++ b/src/cameras/detection/cam_detection.py
@@ -5,7 +5,7 @@ import cv2
from src.cameras.detection.models import FoundCamerasResponse, RawCamera
-CAM_CHECK_NUM = 5
+CAM_CHECK_NUM = 20
logger = logging.getLogger(__name__)
diff --git a/src/core_processor/board_detection/base_pose_estimation.py b/src/core_processor/board_detection/base_pose_estimation.py
index ea9063b4..d0a43174 100644
--- a/src/core_processor/board_detection/base_pose_estimation.py
+++ b/src/core_processor/board_detection/base_pose_estimation.py
@@ -1,9 +1,20 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
import cv2
+import numpy as np
from src.core_processor.board_detection.charuco_constants import aruco_dict, board
+@dataclass
+class CharucoData:
+ charuco_corners: Optional[np.ndarray]
+ charuco_ids: Optional[np.ndarray]
+ aruco_square_corners: Optional[np.ndarray]
+ aruco_square_ids: Optional[np.ndarray]
+
-def detect_charuco_board(image):
+def detect_charuco_board(image) -> CharucoData:
"""
Charuco base pose estimation.
more-or-less copied from - https://mecaruco2.readthedocs.io/en/latest/notebooks_rst/Aruco
@@ -34,4 +45,7 @@ def detect_charuco_board(image):
charuco_corners = res2[1]
charuco_ids = res2[2]
- return charuco_corners, charuco_ids, aruco_square_corners, aruco_square_ids
+ return CharucoData(charuco_corners=charuco_corners,
+ charuco_ids=charuco_ids,
+ aruco_square_corners=aruco_square_corners,
+ aruco_square_ids=aruco_square_ids)
diff --git a/src/core_processor/board_detection/board_detection.py b/src/core_processor/board_detection/board_detection.py
index 36a91f28..67c20ffd 100644
--- a/src/core_processor/board_detection/board_detection.py
+++ b/src/core_processor/board_detection/board_detection.py
@@ -1,6 +1,7 @@
import logging
import time
import traceback
+from dataclasses import dataclass
from pathlib import Path
from types import FunctionType
from typing import Any, Callable, Optional
@@ -11,7 +12,7 @@ import numpy as np
from src.cameras.capture.frame_payload import FramePayload
from src.cameras.multicam_manager.cv_camera_manager import CVCameraManager
from src.cameras.persistence.video_writer.video_writer import SaveOptions
-from src.core_processor.board_detection.base_pose_estimation import detect_charuco_board
+from src.core_processor.board_detection.base_pose_estimation import detect_charuco_board, CharucoData
from src.core_processor.board_detection.charuco_image_annotator import (
annotate_image_with_charuco_data,
)
@@ -21,9 +22,15 @@ from src.core_processor.utils.image_fps_writer import write_fps_to_image
logger = logging.getLogger(__name__)
+@dataclass
+class CharucoFramePayload:
+ raw_frame_payload: FramePayload
+ annotated_frame_payload: FramePayload
+ charuco_data: CharucoData
class BoardDetection:
def __init__(self, cam_manager: CVCameraManager = CVCameraManager()):
+ print('pre-henlo')
self._cam_manager = cam_manager
async def process_by_cam_id(self, webcam_id: str, cb):
@@ -38,14 +45,10 @@ class BoardDetection:
while True:
if not cv_cam.is_capturing_frames:
return
- response = self._process_single_cam_frame(cv_cam)
- if cb and response.image is not None:
- writer.write(
- FramePayload(
- image=response.image, timestamp=response.timestamp
- )
- )
- await cb(response.image)
+ charuco_frame_payload = self._detect_charuco_board_in_image(cv_cam)
+ if cb and charuco_frame_payload.annotated_frame_payload.image is not None:
+ writer.write(charuco_frame_payload.annotated_frame_payload)
+ await cb(charuco_frame_payload.annotated_frame_payload.image)
except Exception as e:
logger.error("Printing traceback")
traceback.print_exc()
@@ -65,7 +68,7 @@ class BoardDetection:
writer.save(options)
def process(
- self, show_window=True, post_processed_frame_cb=Callable[[FramePayload], None]
+ self, post_processed_frame_cb:Callable[[str, CharucoFramePayload], None]=None, show_window=True, save_video=True,
):
"""
Opens Camera using OpenCV and begins image processing for charuco board
@@ -82,19 +85,23 @@ class BoardDetection:
for response in session_obj:
cv_cam = response.cv_cam
writer = response.writer
- payload = self._process_single_cam_frame(cv_cam)
+ charuco_frame_payload = self._detect_charuco_board_in_image(cv_cam)
current_webcam_id = cv_cam.webcam_id_as_str
- if payload is not None:
- writer.write(payload)
- if post_processed_frame_cb:
- post_processed_frame_cb(payload)
+ if not charuco_frame_payload:
+ continue
+
+ if save_video:
+ writer.write(charuco_frame_payload.annotated_frame_payload)
+
+ if post_processed_frame_cb:
+ post_processed_frame_cb(current_webcam_id, charuco_frame_payload)
- fps_manager.increment_frame_processed_for(current_webcam_id)
- if show_window:
- should_continue = show_cam_window(
- current_webcam_id, payload.image, fps_manager
- )
+ fps_manager.increment_frame_processed_for(current_webcam_id)
+ if show_window:
+ should_continue = show_cam_window(
+ current_webcam_id, charuco_frame_payload.annotated_frame_payload.image, fps_manager
+ )
except:
logger.error("Printing traceback")
traceback.print_exc()
@@ -117,28 +124,32 @@ class BoardDetection:
cv2.destroyWindow(cv_cam.webcam_id_as_str)
cv2.waitKey(1)
- def _process_single_cam_frame(self, cv_cam):
- success, frame, timestamp = cv_cam.latest_frame
- if not success:
+ def _detect_charuco_board_in_image(self, cv_cam):
+ raw_frame_payload = cv_cam.latest_frame
+
+ if not raw_frame_payload.success:
# logger.error("CV2 failed to grab a frame")
return
- if frame is None:
+ if raw_frame_payload.image is None:
logger.error("Frame is empty")
return
- (
- charuco_corners,
- charuco_ids,
- aruco_square_corners,
- aruco_square_ids,
- ) = detect_charuco_board(frame)
- # TODO - Pull out timestamps per frame and calculate fps to display on image
+ charuco_data = detect_charuco_board(raw_frame_payload.image)
+
+ annotated_image = raw_frame_payload.image.copy()
+
success_bool = annotate_image_with_charuco_data(
- frame, cv_cam.webcam_id_as_str, charuco_corners, charuco_ids
+ annotated_image, cv_cam.webcam_id_as_str, charuco_data
+ )
+ cv2.polylines(annotated_image, np.int32([charuco_data.charuco_corners]), True, (0, 100, 255), 2)
+
+ return CharucoFramePayload(
+ annotated_frame_payload=FramePayload(image=annotated_image, timestamp=time.time_ns()),
+ raw_frame_payload=raw_frame_payload,
+ charuco_data=charuco_data
)
- cv2.polylines(frame, np.int32([charuco_corners]), True, (0, 100, 255), 2)
- return FramePayload(image=frame, timestamp=time.time_ns())
if __name__ == "__main__":
## Jon, Run me to easily start a charuco board detect run
+ print('starting main')
BoardDetection().process()
diff --git a/src/core_processor/board_detection/charuco_image_annotator.py b/src/core_processor/board_detection/charuco_image_annotator.py
index 116d763d..ed183c7b 100644
--- a/src/core_processor/board_detection/charuco_image_annotator.py
+++ b/src/core_processor/board_detection/charuco_image_annotator.py
@@ -22,15 +22,18 @@ def _board_text(
def annotate_image_with_charuco_data(
- image, webcam_id: str, charuco_corners, charuco_ids
+ image, webcam_id: str, charuco_data
) -> bool:
full_charuco_detected_on_this_frame = False
- if len(charuco_ids) == num_charuco_corners:
+ if len(charuco_data.charuco_ids) == num_charuco_corners:
full_charuco_detected_on_this_frame = True
image_w_markers = cv2.aruco.drawDetectedCornersCharuco(
- image, np.array(charuco_corners), np.array(charuco_ids), (0, 255, 125, 255)
+ image,
+ np.array(charuco_data.charuco_corners),
+ np.array(charuco_data.charuco_ids),
+ (0, 255, 125, 255)
) # yellow? I
# think cv2 uses BGR instead of RGB?
@@ -39,7 +42,7 @@ def annotate_image_with_charuco_data(
text_to_write_on_this_camera = ""
current_cam_corner_count_str = _board_text(
this_cam_name,
- charuco_ids,
+ charuco_data.charuco_ids,
num_charuco_corners,
full_charuco_detected_on_this_frame,
)
@@ -76,7 +79,7 @@ def annotate_image_with_charuco_data(
if full_charuco_detected_on_this_frame:
cv2.polylines(
- image_w_markers, np.int32([charuco_corners]), False, (0, 100, 255), 2
+ image_w_markers, np.int32([charuco_data.charuco_corners]), False, (0, 100, 255), 2
)
# for these_corners in charuco_points_from_previous_frames:
# if len(these_corners)>0:
diff --git a/src/core_processor/camera_calibration/camera_calibration.py b/src/core_processor/camera_calibration/camera_calibration.py
index 14004ba0..06952714 100644
--- a/src/core_processor/camera_calibration/camera_calibration.py
+++ b/src/core_processor/camera_calibration/camera_calibration.py
@@ -1,5 +1,5 @@
from src.cameras.capture.frame_payload import FramePayload
-from src.core_processor.board_detection.board_detection import BoardDetection
+from src.core_processor.board_detection.board_detection import BoardDetection, CharucoFramePayload
class CameraCalibration:
@@ -9,14 +9,16 @@ class CameraCalibration:
def perform_calibration(self):
self._bd.process(
show_window=True,
+ save_video=True,
post_processed_frame_cb=self._calibration_work,
)
- def _calibration_work(self, frame: FramePayload):
- print("found frame", frame.timestamp)
+ def _calibration_work(self, webcam_id: str, charuco_frame_payload: CharucoFramePayload):
+ print(f"found frame from camera {webcam_id} at timestamp {charuco_frame_payload.annotated_frame_payload.timestamp/1e9}")
if __name__ == "__main__":
+ print('start main')
a = BoardDetection()
b = CameraCalibration(a)
b.perform_calibration()