Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

draw charuco data on setup video

jon matthise33a069ef100
diff --git a/freemocap/calibrate.py b/freemocap/calibrate.py
index bb44147a..76c65370 100644
--- a/freemocap/calibrate.py
+++ b/freemocap/calibrate.py
@@ -62,52 +62,7 @@ def CalibrateCaptureVolume(session,board, calVideoFrameLength = .5):
session.cameraCalFilePath = session.sessionPath / calibrationFile
error,charuco_data, charuco_frames = cgroup.calibrate_videos(vidnames, board)
-
- ## iteratively retry if anipose fails, butcouldn't get this to work because anipose launches some weird thread process that doesn't terminate properly?
- # try:
- # error,charuco_data, charuco_frames = cgroup.calibrate_videos(vidnames, board)
- # if not spoof_anipose_fail_bool:
- # anipose_success = True
- # else:
- # anipose_success = False
- # except:
- # anipose_success = False
- # console.rule('Anipose Calibration Failed for frame range {} - {}'.format(calVideoFrameLength[0], calVideoFrameLength[1]), style="color({})".format(3))
-
- # anipose_iter = -1
- # if not anipose_success: #%% run anipose calibration, and if it fails, re-make calibration vids with different frame range and try again
-
- # frame_window_size = round(session.numFrames*.5)
- # cal_video_frame_range = [0, frame_window_size]
- # while not anipose_success:
- # anipose_iter += 1
- # console.rule('Anipose Failed - Reprocessing - Iteration #{}'.format(anipose_iter), style="color({})".format(3))
-
- # if cal_video_frame_range[1] > session.numFrames:
- # console.rule('We moved the window all the way through the videos and Anipose still isn\'t happy, starting again from the begining with a shorter frame window', style="color({})".format(3))
- # frame_window_size = round(frame_window_size/2)
- # cal_video_frame_range = [0, frame_window_size]
-
- # if round(cal_video_frame_range[-1] * session.numFrames) < 1 or anipose_iter > 20:
- # Exception('Sorry, we weren\'t able to get AniPose to run successfully :( - Make sure your Charuco board was visible to each camera (not necessarily at the same time) and that there wasn\'t, like, a bunch of glare on it')
- # return
-
- # console.rule('Trying Anipose calibration again with frame range {} - {}'.format(cal_video_frame_range[0], cal_video_frame_range[1]), style="color({})".format(3))
- # createCalibrationVideos(session, cal_video_frame_range)
-
- # try:
- # error,charuco_data, charuco_frames = cgroup.calibrate_videos(vidnames, board)
-
- # if not spoof_anipose_fail_bool:
- # anipose_success=True
- # else:
- # cal_video_frame_range[0] = cal_video_frame_range[0] + round(session.numFrames*.1)
- # cal_video_frame_range[1] = cal_video_frame_range[1] + round(session.numFrames*.1)
- # except:
- # cal_video_frame_range[0] = cal_video_frame_range[0] + round(session.numFrames*.1)
- # cal_video_frame_range[1] = cal_video_frame_range[1] + round(session.numFrames*.1)
-
-
+
cgroup.dump(session.cameraCalFilePath)
camera_calibration_info_dict = cgroup.get_dicts()
diff --git a/freemocap/webcam/camsetup.py b/freemocap/webcam/camsetup.py
index c29af680..33a7bbde 100644
--- a/freemocap/webcam/camsetup.py
+++ b/freemocap/webcam/camsetup.py
@@ -8,6 +8,7 @@ import time
import mediapipe as mp
import numpy as np
+
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_holistic = mp.solutions.holistic
@@ -63,6 +64,9 @@ class VideoSetup(threading.Thread):
while True:
ret1, frame1 = cap.read()
+ charuco_corners, charuco_ids, aruco_square_corners, aruco_square_ids = detect_charuco_board(frame1)
+ annotate_image_with_charuco_data(frame1, charuco_corners, charuco_ids)
+
if ret1 == True:
if rotNum is not None:
frame1 = imutils.rotate_bound(frame1, angle=rotNum)
@@ -105,7 +109,7 @@ class MediaPipeVideoSetup(threading.Thread):
with mp_holistic.Holistic(
static_image_mode=False,
- model_complexity=2,
+ model_complexity=0,
enable_segmentation=True) as holistic:
cap.set(cv2.CAP_PROP_FRAME_WIDTH, resWidth)
@@ -137,6 +141,15 @@ class MediaPipeVideoSetup(threading.Thread):
print(f"mean fps for cam {self.camID} is {1/np.mean(np.diff(timestamps))}")
if ret1 == True:
+
+ charuco_corners, charuco_ids, aruco_square_corners, aruco_square_ids = detect_charuco_board(frame1)
+ try:
+ results = holistic.process(cv2.cvtColor(frame1, cv2.COLOR_BGR2RGB))
+ except Exception as e:
+ print(e)
+
+ annotate_image_with_charuco_data(frame1, charuco_corners, charuco_ids)
+
frame1.flags.writeable = False
try:
results = holistic.process(cv2.cvtColor(frame1, cv2.COLOR_BGR2RGB))
@@ -169,6 +182,10 @@ class MediaPipeVideoSetup(threading.Thread):
.get_default_hand_landmarks_style())
if rotNum is not None:
frame1 = imutils.rotate_bound(frame1, angle=rotNum)
+
+
+
+
cv2.imshow(camWindowName, frame1)
if cv2.waitKey(1) & 0xFF == 27:
# == ord('q') for q
@@ -199,3 +216,99 @@ def RunSetup(cam_inputs, rotation_input, paramDict,mediaPipeOverlay):
for k in ulist:
k.join()
+
+
+
+
+
+def detect_charuco_board(image, annotate_image=True):
+ """
+ Charuco base pose estimation.
+ more-or-less copied from - https://mecaruco2.readthedocs.io/en/latest/notebooks_rst/Aruco
+ /sandbox/ludovic/aruco_calibration_rotation.html
+ """
+ charuco_corners = []
+ charuco_ids = []
+
+ aruco_dict = cv2.aruco.Dictionary_get(cv2.aruco.DICT_4X4_250)
+ charuco_length = 7
+ charuco_width = 5
+
+ board = cv2.aruco.CharucoBoard_create(charuco_length, charuco_width, 1, .8, aruco_dict)
+ global num_charuco_corners
+ num_charuco_corners = (charuco_length-1) * (charuco_width-1)
+
+
+ # SUB PIXEL CORNER DETECTION CRITERION
+ criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 0.00001)
+
+ gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+ aruco_square_corners, aruco_square_ids, rejectedImgPoints = cv2.aruco.detectMarkers(gray,
+ aruco_dict)
+
+ if len(aruco_square_corners) > 0:
+ # SUB PIXEL DETECTION
+ for this_corner in aruco_square_corners:
+ cv2.cornerSubPix(gray, this_corner,
+ winSize=(3, 3),
+ zeroZone=(-1, -1),
+ criteria=criteria)
+ res2 = cv2.aruco.interpolateCornersCharuco(aruco_square_corners, aruco_square_ids, gray,
+ board)
+
+ if res2[1] is not None and res2[2] is not None and len(res2[1]) > 3:
+ charuco_corners = res2[1]
+ charuco_ids = res2[2]
+
+
+ return charuco_corners, charuco_ids, aruco_square_corners, aruco_square_ids
+
+def annotate_image_with_charuco_data(image, charuco_corners, charuco_ids)->bool:
+
+ full_charuco_detected_on_this_frame = False
+ if len(charuco_ids) == num_charuco_corners:
+ full_charuco_detected_on_this_frame = True
+
+ image_w_markers = cv2.aruco.drawDetectedCornersCharuco(image,
+ np.array(charuco_corners),
+ np.array(charuco_ids),
+ (200,100,200,255)) #I think cv2 uses BGR instead of RGB?
+
+
+ text_to_write_on_this_camera = ''
+ current_cam_corner_count_str = str(
+ len(charuco_ids)) + " of " + str(
+ num_charuco_corners) + " ChAruco Corner Points detected | Full Board Detected: " + str(
+ full_charuco_detected_on_this_frame)
+ # TODO - Determine 'shared views' (i.e. frames in which a full board is detected by 2 cameras)
+ # TODO - self.determine_shared_charuco_board_views()
+ # this_cam_shared_views_str = " | Shared Views: " + str(
+ # each_cameras_shared_board_view_count_total)
+ text_to_write_on_this_camera = current_cam_corner_count_str
+
+ position = (10, 50)
+ cv2.putText(
+ image_w_markers, # numpy array on which text is written
+ text_to_write_on_this_camera, # text
+ position, # position at which writing has to start
+ cv2.FONT_HERSHEY_SIMPLEX, # font family
+ .5, # font size
+ (0, 10, 0, 255), # font color
+ 4) # font stroke (draw a darker heavier font beneath a lighter/thinner copy for readability)
+
+ cv2.putText(
+ image_w_markers, # numpy array on which text is written
+ text_to_write_on_this_camera, # text
+ position, # position at which writing has to start
+ cv2.FONT_HERSHEY_SIMPLEX, # font family (very limited selection, i think there's some interesting CV history here...)
+ .5, # font size
+ (209, 180, 0, 255), # font color
+ 2 ) # font stroke
+
+ if full_charuco_detected_on_this_frame:
+ cv2.polylines(image_w_markers, np.int32([charuco_corners]), True, (0, 255, 255), 4)
+ # for these_corners in charuco_points_from_previous_frames:
+ # if len(these_corners)>0:
+ # cv2.polylines(image_w_markers, np.int32([these_corners]), True, (0,100,255,255/2), 2)
+
+ return True
\ No newline at end of file