Free Motion Capture for Everyone πβ¨
TypeScript
Threading implementation working queue-based IPC is a little too slow
Endurance Idehenf15602da7a47
diff --git a/jon_scratch/opencv_camera.py b/jon_scratch/opencv_camera.pyindex 1104a3b8..e5e27733 100644--- a/jon_scratch/opencv_camera.py+++ b/jon_scratch/opencv_camera.py@@ -1,10 +1,12 @@import loggingimport platform+import threadingimport time+from typing import Any, Listimport cv2import numpy as np-from pydantic import BaseModel+from pydantic import BaseModel, PrivateAttrlogger = logging.getLogger(__name__)logger.level = logging.INFO@@ -35,6 +37,10 @@ class OpenCVCamera(TweakedModel):# resolution_height: int = 720opencv_video_capture_object: cv2.VideoCapture = None+ _is_capturing_frames: bool = PrivateAttr(False)+ _running_thread = PrivateAttr(None)+ _last_100_frames: List[Any] = PrivateAttr([])+def connect(self):if platform.system() == 'Windows':cap_backend = cv2.CAP_DSHOW@@ -50,18 +56,41 @@ class OpenCVCamera(TweakedModel):return success# set camera stream parameters- self.opencv_video_capture_object.set(cv2.CAP_PROP_EXPOSURE, self.exposure)- self.opencv_video_capture_object.set(cv2.CAP_PROP_FRAME_WIDTH, self.resolution_width)- self.opencv_video_capture_object.set(cv2.CAP_PROP_FRAME_HEIGHT, self.resolution_height)-- self.opencv_video_capture_object.set(cv2.CAP_PROP_FOURCC,- cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'))+ # self.opencv_video_capture_object.set(cv2.CAP_PROP_EXPOSURE, self.exposure)+ # self.opencv_video_capture_object.set(cv2.CAP_PROP_FRAME_WIDTH, self.resolution_width)+ # self.opencv_video_capture_object.set(cv2.CAP_PROP_FRAME_HEIGHT, self.resolution_height)+ #+ # self.opencv_video_capture_object.set(cv2.CAP_PROP_FOURCC,+ # cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'))if success:logger.debug(f'Camera found at port number {self.port_number}')self.name = f'Camera {self.port_number}'return success+ def start_frame_capture(self):+ t = threading.Thread(+ target=self._start_frame_loop+ )+ t.daemon = True+ t.start()+ self._running_thread = t++ def stop_frame_capture(self):+ self._is_capturing_frames = False+ print("Thread should be stopped")++ def latest_frame(self):+ if self._last_100_frames:+ return self._last_100_frames[-1]+ return False, None, None++ def _start_frame_loop(self):+ self._is_capturing_frames = True+ while self._is_capturing_frames:+ success, image, timestamp = self.get_next_frame()+ self._last_100_frames.append((success, image, timestamp))+def get_next_frame(self):timestamp_ns_pre_grab = time.time_ns()# Why grab not read? see ->diff --git a/src/api/routes/camera/cam_data_ws.py b/src/api/routes/camera/cam_data_ws.pyindex 6d824ada..b61264ae 100644--- a/src/api/routes/camera/cam_data_ws.py+++ b/src/api/routes/camera/cam_data_ws.py@@ -1,7 +1,6 @@import loggingfrom time import perf_counter-import aiomultiprocessimport orjsonfrom fastapi import APIRouter, WebSocket@@ -24,6 +23,7 @@ async def begin_board_detection():service = BoardDetectService()await service.run()+@cam_ws_router.get("/begin_mediapipe_skeleton_detection")async def begin_mediapipe_skeleton_detection(model_complexity=2):"""diff --git a/src/api/services/board_detect_service.py b/src/api/services/board_detect_service.pyindex da5bfaef..eaff247c 100644--- a/src/api/services/board_detect_service.py+++ b/src/api/services/board_detect_service.py@@ -1,18 +1,7 @@-import asyncio--from aiomultiprocess.core import get_manager-from src.core_processor.board_detection.detect import BoardDetection-from src.core_processor.processor import start_camera_captureclass BoardDetectService:async def run(self):- queue = get_manager().Queue()- await asyncio.gather(- # Producer- start_camera_capture(queue),- # Consumer- BoardDetection().create_new_process_for_run(queue)- )+ await BoardDetection().process()diff --git a/src/api/services/mediapipe_detect_service.py b/src/api/services/mediapipe_detect_service.pyindex ad36fe27..8fe38027 100644--- a/src/api/services/mediapipe_detect_service.py+++ b/src/api/services/mediapipe_detect_service.py@@ -8,7 +8,7 @@ from src.core_processor.processor import start_camera_captureclass MediapipeSkeletonDetectionService:- model_complexity: int=1+ model_complexity: int = 1async def run(self):queue = get_manager().Queue()diff --git a/src/core_processor/board_detection/detect.py b/src/core_processor/board_detection/detect.pyindex b6097464..20608ae1 100644--- a/src/core_processor/board_detection/detect.py+++ b/src/core_processor/board_detection/detect.py@@ -1,61 +1,53 @@+import asyncioimport loggingimport cv2import numpy as np-from aiomultiprocess import Process-from aiomultiprocess.types import Queue--from src.core_processor.processor import ImagePayload---from rich import inspect-+from src.core_processor.processor import create_opencv_camsaruco_dict = cv2.aruco.Dictionary_get(cv2.aruco.DICT_4X4_250)charuco_length = 7charuco_width = 5board = cv2.aruco.CharucoBoard_create(charuco_length, charuco_width, 1, .8, aruco_dict)-num_charuco_corners = (charuco_length-1) * (charuco_width-1)+num_charuco_corners = (charuco_length - 1) * (charuco_width - 1)class BoardDetection:- async def create_new_process_for_run(self, queue: Queue):- p = Process(target=self.process, args=(queue,))- p.start()- await p.join()-- async def process(self, queue: Queue):+ async def process(self):logger = logging.getLogger(__name__)- while True:- message = None- try:- message = queue.get(timeout=1) # type: ImagePayload-- except Exception as e:- pass--- if not message:- continue-- frames = message.frames+ cv_cams = create_opencv_cams()- for f in frames:- print(f'got image of shape {f.image.shape} from camera at port {f.port_number}, queueue size: {queue.qsize()}')- charuco_corners, charuco_ids, aruco_square_corners, aruco_square_ids = self.detect_charuco_board(f.image)- #TODO - Pull out timestamps per frame and calculate fps to display on image- success_bool = self.annotate_image_with_charuco_data(f.image, f.port_number, charuco_corners, charuco_ids)+ for cv_cam in cv_cams:+ cv_cam.start_frame_capture()- cv2.polylines(f.image, np.int32([charuco_corners]), True, (0,100,255), 2)- cv2.imshow(str(f.port_number), f.image)+ while True:+ for cv_cam in cv_cams:+ success, frame, timestamp = cv_cam.latest_frame()+ if not success:+ continue++ if frame is None:+ continue++ port_number = str(cv_cam.port_number)+ print(+ f'got image of shape {frame.shape} from camera at port {port_number}')+ charuco_corners, charuco_ids, aruco_square_corners, aruco_square_ids = \+ self.detect_charuco_board(frame)+ # TODO - Pull out timestamps per frame and calculate fps to display on image+ success_bool = self.annotate_image_with_charuco_data(frame, port_number,+ charuco_corners, charuco_ids)++ cv2.polylines(frame, np.int32([charuco_corners]), True, (0, 100, 255), 2)++ cv2.imshow(port_number, frame)exit_key = cv2.waitKey(1)if exit_key == 27:break-def detect_charuco_board(self, image):"""Charuco base pose estimation.@@ -64,14 +56,14 @@ class BoardDetection:"""charuco_corners = []charuco_ids = []-+# SUB PIXEL CORNER DETECTION CRITERIONcriteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 0.00001)-+gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)aruco_square_corners, aruco_square_ids, rejectedImgPoints = cv2.aruco.detectMarkers(gray,aruco_dict)-+if len(aruco_square_corners) > 0:# SUB PIXEL DETECTIONfor this_corner in aruco_square_corners:@@ -81,23 +73,25 @@ class BoardDetection:criteria=criteria)res2 = cv2.aruco.interpolateCornersCharuco(aruco_square_corners, aruco_square_ids, gray,board)-+if res2[1] is not None and res2[2] is not None and len(res2[1]) > 3:charuco_corners = res2[1]charuco_ids = res2[2]-+return charuco_corners, charuco_ids, aruco_square_corners, aruco_square_ids- def annotate_image_with_charuco_data(self, image, port_number, charuco_corners, charuco_ids)->bool:+ def annotate_image_with_charuco_data(self, image, port_number, charuco_corners,+ charuco_ids) -> bool:full_charuco_detected_on_this_frame = Falseif len(charuco_ids) == num_charuco_corners:full_charuco_detected_on_this_frame = Trueimage_w_markers = cv2.aruco.drawDetectedCornersCharuco(image,- np.array(charuco_corners),- np.array(charuco_ids),- (0,255,125,255)) #yellow? I think cv2 uses BGR instead of RGB?+ np.array(charuco_corners),+ np.array(charuco_ids),+ (0, 255, 125, 255)) # yellow? I+ # think cv2 uses BGR instead of RGB?this_cam_name = "Camera " + str(port_number)@@ -106,7 +100,8 @@ class BoardDetection:len(charuco_ids)) + " of " + str(num_charuco_corners) + " ChAruco Corner Points detected | Full Board Detected: " + str(full_charuco_detected_on_this_frame)- # TODO - Determine 'shared views' (i.e. frames in which a full board is detected by 2 cameras)+ # TODO - Determine 'shared views' (i.e. frames in which a full board is detected by 2+ # cameras)# TODO - self.determine_shared_charuco_board_views()# this_cam_shared_views_str = " | Shared Views: " + str(# each_cameras_shared_board_view_count_total)@@ -120,26 +115,29 @@ class BoardDetection:cv2.FONT_HERSHEY_SIMPLEX, # font family.5, # font size(30, 10, 0, 255), # font color- 2) # font stroke (draw a darker heavier font beneath a lighter/thinner copy for readability)+ 2) # font stroke (draw a darker heavier font beneath a lighter/thinner copy for+ # readability)cv2.putText(image_w_markers, # numpy array on which text is writtentext_to_write_on_this_camera, # textposition, # position at which writing has to start- cv2.FONT_HERSHEY_SIMPLEX, # font family (very limited selection, i think there's some interesting CV history here...)+ cv2.FONT_HERSHEY_SIMPLEX,+ # font family (very limited selection, i think there's some interesting CV history+ # here...).5, # font size(209, 180, 0, 255), # font color- 1 ) # font stroke+ 1) # font strokeif full_charuco_detected_on_this_frame:cv2.polylines(image_w_markers, np.int32([charuco_corners]), False, (0, 100, 255), 2)# for these_corners in charuco_points_from_previous_frames:# if len(these_corners)>0:- # cv2.polylines(image_w_markers, np.int32([these_corners]), True, (0,100,255,255/2), 2)+ # cv2.polylines(image_w_markers, np.int32([these_corners]), True, (0,100,255,+ # 255/2), 2)return True-def determine_shared_charuco_board_views(self):pass# # determine paired board views@@ -148,14 +146,21 @@ class BoardDetection:# for this_other_camera_num in range(multi_cam.num_cams):# if this_other_camera_num != this_cam_num:# if full_charuco_detected_on_this_frame[this_other_camera_num]:- # each_cameras_shared_board_view_count_array[this_cam_num, this_other_camera_num] += 1- # each_cameras_shared_board_view_count_total = np.sum(each_cameras_shared_board_view_count_array, axis=1)+ # each_cameras_shared_board_view_count_array[this_cam_num,+ # this_other_camera_num] += 1+ # each_cameras_shared_board_view_count_total = np.sum(+ # each_cameras_shared_board_view_count_array, axis=1)# # print(each_cameras_shared_board_view_count)## for this_pair_num, this_cam_pair in enumerate(camera_pair_list):# camera_one_id = this_cam_pair[0]# camera_two_id = this_cam_pair[1]#- # if full_charuco_detected_on_this_frame[camera_one_id] and full_charuco_detected_on_this_frame[+ # if full_charuco_detected_on_this_frame[camera_one_id] and+ # full_charuco_detected_on_this_frame[# camera_two_id]:- # camera_pair_joint_view_count[this_pair_num] += 1\ No newline at end of file+ # camera_pair_joint_view_count[this_pair_num] += 1+++if __name__ == "__main__":+ asyncio.run(BoardDetection().process())diff --git a/src/core_processor/processor.py b/src/core_processor/processor.pyindex c64c4988..e0d23116 100644--- a/src/core_processor/processor.py+++ b/src/core_processor/processor.py@@ -40,7 +40,7 @@ class FramePayload(NamedTuple):class ImagePayload(NamedTuple):- frames: List[FramePayload]+ frames: List[FramePayload] = Noneclass CameraCaptureProcess:@@ -68,7 +68,7 @@ class CameraCaptureProcess:))- _queue.put(+ _queue.put_nowait(ImagePayload(frames=image_list))diff --git a/src/core_processor/testing/threading_single_cam.py b/src/core_processor/testing/threading_single_cam.pynew file mode 100644index 00000000..39b3fbec--- /dev/null+++ b/src/core_processor/testing/threading_single_cam.py@@ -0,0 +1,100 @@+import time+from threading import Thread++import cv2+++class WebcamStream:+ def __init__(self, stream_id=0):+ self.stream_id = stream_id # default is 0 for primary camera++ # opening video capture stream+ self.vcap = cv2.VideoCapture(self.stream_id)+ if self.vcap.isOpened() is False:+ print("[Exiting]: Error accessing webcam stream.")+ exit(0)+ fps_input_stream = int(self.vcap.get(5))+ print("FPS of webcam hardware/input stream: {}".format(fps_input_stream))++ # reading a single frame from vcap stream for initializing+ self.grabbed, self.frame = self.vcap.read()+ if self.grabbed is False:+ print('[Exiting] No more frames to read')+ exit(0)++ # self.stopped is set to False when frames are being read from self.vcap stream+ self.stopped = True++ # reference to the thread for reading next available frame from input stream+ self.t = Thread(target=self.update, args=())+ self.t.daemon = True # daemon threads keep running in the background while the program+ # is executing++ # method for starting the thread for grabbing next available frame in input stream+ def start(self):+ self.stopped = False+ self.t.start()++ # method for reading next frame++ def update(self):+ while True:+ if self.stopped is True:+ break+ self.grabbed, self.frame = self.vcap.read()+ if self.grabbed is False:+ print('[Exiting] No more frames to read')+ self.stopped = True+ break+ self.vcap.release()++ # method for returning latest read frame+ def read(self):+ return self.frame++ # method called to stop reading frames+ def stop(self):+ self.stopped = True++ # initializing and starting multi-threaded webcam capture input stream+++webcam_stream1 = WebcamStream(stream_id=0) # stream_id = 0 is for primary camera+webcam_stream2 = WebcamStream(stream_id=1) # stream_id = 0 is for primary camera+webcam_stream1.start()+webcam_stream2.start()++# processing frames in input stream+num_frames_processed = 0+start = time.time()+while True:+ if webcam_stream1.stopped is True:+ break+ if webcam_stream2.stopped is True:+ break+ else:+ frame1 = webcam_stream1.read()+ frame2 = webcam_stream2.read()++ # adding a delay for simulating time taken for processing a frame+ delay = 0.03 # delay value in seconds. so, delay=1 is equivalent to 1 second+ time.sleep(delay)+ num_frames_processed += 1++ cv2.imshow('frame1', frame1)+ cv2.imshow('frame2', frame2)+ key = cv2.waitKey(1)+ if key == ord('q'):+ break+end = time.time()+webcam_stream1.stop() # stop the webcam stream+webcam_stream2.stop() # stop the webcam stream++# printing time elapsed and fps+elapsed = end - start+fps = num_frames_processed / elapsed+print(+ "FPS: {} , Elapsed Time: {} , Frames Processed: {}".format(fps, elapsed, num_frames_processed))++# closing all windows+cv2.destroyAllWindows()