Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

Threading implementation working queue-based IPC is a little too slow

Endurance Idehenf15602da7a47
diff --git a/jon_scratch/opencv_camera.py b/jon_scratch/opencv_camera.py
index 1104a3b8..e5e27733 100644
--- a/jon_scratch/opencv_camera.py
+++ b/jon_scratch/opencv_camera.py
@@ -1,10 +1,12 @@
import logging
import platform
+import threading
import time
+from typing import Any, List
import cv2
import numpy as np
-from pydantic import BaseModel
+from pydantic import BaseModel, PrivateAttr
logger = logging.getLogger(__name__)
logger.level = logging.INFO
@@ -35,6 +37,10 @@ class OpenCVCamera(TweakedModel):
# resolution_height: int = 720
opencv_video_capture_object: cv2.VideoCapture = None
+ _is_capturing_frames: bool = PrivateAttr(False)
+ _running_thread = PrivateAttr(None)
+ _last_100_frames: List[Any] = PrivateAttr([])
+
def connect(self):
if platform.system() == 'Windows':
cap_backend = cv2.CAP_DSHOW
@@ -50,18 +56,41 @@ class OpenCVCamera(TweakedModel):
return success
# set camera stream parameters
- self.opencv_video_capture_object.set(cv2.CAP_PROP_EXPOSURE, self.exposure)
- self.opencv_video_capture_object.set(cv2.CAP_PROP_FRAME_WIDTH, self.resolution_width)
- self.opencv_video_capture_object.set(cv2.CAP_PROP_FRAME_HEIGHT, self.resolution_height)
-
- self.opencv_video_capture_object.set(cv2.CAP_PROP_FOURCC,
- cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'))
+ # self.opencv_video_capture_object.set(cv2.CAP_PROP_EXPOSURE, self.exposure)
+ # self.opencv_video_capture_object.set(cv2.CAP_PROP_FRAME_WIDTH, self.resolution_width)
+ # self.opencv_video_capture_object.set(cv2.CAP_PROP_FRAME_HEIGHT, self.resolution_height)
+ #
+ # self.opencv_video_capture_object.set(cv2.CAP_PROP_FOURCC,
+ # cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'))
if success:
logger.debug(f'Camera found at port number {self.port_number}')
self.name = f'Camera {self.port_number}'
return success
+ def start_frame_capture(self):
+ t = threading.Thread(
+ target=self._start_frame_loop
+ )
+ t.daemon = True
+ t.start()
+ self._running_thread = t
+
+ def stop_frame_capture(self):
+ self._is_capturing_frames = False
+ print("Thread should be stopped")
+
+ def latest_frame(self):
+ if self._last_100_frames:
+ return self._last_100_frames[-1]
+ return False, None, None
+
+ def _start_frame_loop(self):
+ self._is_capturing_frames = True
+ while self._is_capturing_frames:
+ success, image, timestamp = self.get_next_frame()
+ self._last_100_frames.append((success, image, timestamp))
+
def get_next_frame(self):
timestamp_ns_pre_grab = time.time_ns()
# Why grab not read? see ->
diff --git a/src/api/routes/camera/cam_data_ws.py b/src/api/routes/camera/cam_data_ws.py
index 6d824ada..b61264ae 100644
--- a/src/api/routes/camera/cam_data_ws.py
+++ b/src/api/routes/camera/cam_data_ws.py
@@ -1,7 +1,6 @@
import logging
from time import perf_counter
-import aiomultiprocess
import orjson
from fastapi import APIRouter, WebSocket
@@ -24,6 +23,7 @@ async def begin_board_detection():
service = BoardDetectService()
await service.run()
+
@cam_ws_router.get("/begin_mediapipe_skeleton_detection")
async def begin_mediapipe_skeleton_detection(model_complexity=2):
"""
diff --git a/src/api/services/board_detect_service.py b/src/api/services/board_detect_service.py
index da5bfaef..eaff247c 100644
--- a/src/api/services/board_detect_service.py
+++ b/src/api/services/board_detect_service.py
@@ -1,18 +1,7 @@
-import asyncio
-
-from aiomultiprocess.core import get_manager
-
from src.core_processor.board_detection.detect import BoardDetection
-from src.core_processor.processor import start_camera_capture
class BoardDetectService:
async def run(self):
- queue = get_manager().Queue()
- await asyncio.gather(
- # Producer
- start_camera_capture(queue),
- # Consumer
- BoardDetection().create_new_process_for_run(queue)
- )
+ await BoardDetection().process()
diff --git a/src/api/services/mediapipe_detect_service.py b/src/api/services/mediapipe_detect_service.py
index ad36fe27..8fe38027 100644
--- a/src/api/services/mediapipe_detect_service.py
+++ b/src/api/services/mediapipe_detect_service.py
@@ -8,7 +8,7 @@ from src.core_processor.processor import start_camera_capture
class MediapipeSkeletonDetectionService:
- model_complexity: int=1
+ model_complexity: int = 1
async def run(self):
queue = get_manager().Queue()
diff --git a/src/core_processor/board_detection/detect.py b/src/core_processor/board_detection/detect.py
index b6097464..20608ae1 100644
--- a/src/core_processor/board_detection/detect.py
+++ b/src/core_processor/board_detection/detect.py
@@ -1,61 +1,53 @@
+import asyncio
import logging
import cv2
import numpy as np
-from aiomultiprocess import Process
-from aiomultiprocess.types import Queue
-
-from src.core_processor.processor import ImagePayload
-
-
-from rich import inspect
-
+from src.core_processor.processor import create_opencv_cams
aruco_dict = cv2.aruco.Dictionary_get(cv2.aruco.DICT_4X4_250)
charuco_length = 7
charuco_width = 5
board = cv2.aruco.CharucoBoard_create(charuco_length, charuco_width, 1, .8, aruco_dict)
-num_charuco_corners = (charuco_length-1) * (charuco_width-1)
+num_charuco_corners = (charuco_length - 1) * (charuco_width - 1)
class BoardDetection:
- async def create_new_process_for_run(self, queue: Queue):
- p = Process(target=self.process, args=(queue,))
- p.start()
- await p.join()
-
- async def process(self, queue: Queue):
+ async def process(self):
logger = logging.getLogger(__name__)
- while True:
- message = None
- try:
- message = queue.get(timeout=1) # type: ImagePayload
-
- except Exception as e:
- pass
-
-
- if not message:
- continue
-
- frames = message.frames
+ cv_cams = create_opencv_cams()
- for f in frames:
- print(f'got image of shape {f.image.shape} from camera at port {f.port_number}, queueue size: {queue.qsize()}')
- charuco_corners, charuco_ids, aruco_square_corners, aruco_square_ids = self.detect_charuco_board(f.image)
- #TODO - Pull out timestamps per frame and calculate fps to display on image
- success_bool = self.annotate_image_with_charuco_data(f.image, f.port_number, charuco_corners, charuco_ids)
+ for cv_cam in cv_cams:
+ cv_cam.start_frame_capture()
- cv2.polylines(f.image, np.int32([charuco_corners]), True, (0,100,255), 2)
- cv2.imshow(str(f.port_number), f.image)
+ while True:
+ for cv_cam in cv_cams:
+ success, frame, timestamp = cv_cam.latest_frame()
+ if not success:
+ continue
+
+ if frame is None:
+ continue
+
+ port_number = str(cv_cam.port_number)
+ print(
+ f'got image of shape {frame.shape} from camera at port {port_number}')
+ charuco_corners, charuco_ids, aruco_square_corners, aruco_square_ids = \
+ self.detect_charuco_board(frame)
+ # TODO - Pull out timestamps per frame and calculate fps to display on image
+ success_bool = self.annotate_image_with_charuco_data(frame, port_number,
+ charuco_corners, charuco_ids)
+
+ cv2.polylines(frame, np.int32([charuco_corners]), True, (0, 100, 255), 2)
+
+ cv2.imshow(port_number, frame)
exit_key = cv2.waitKey(1)
if exit_key == 27:
break
-
def detect_charuco_board(self, image):
"""
Charuco base pose estimation.
@@ -64,14 +56,14 @@ class BoardDetection:
"""
charuco_corners = []
charuco_ids = []
-
+
# SUB PIXEL CORNER DETECTION CRITERION
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 0.00001)
-
+
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
aruco_square_corners, aruco_square_ids, rejectedImgPoints = cv2.aruco.detectMarkers(gray,
aruco_dict)
-
+
if len(aruco_square_corners) > 0:
# SUB PIXEL DETECTION
for this_corner in aruco_square_corners:
@@ -81,23 +73,25 @@ class BoardDetection:
criteria=criteria)
res2 = cv2.aruco.interpolateCornersCharuco(aruco_square_corners, aruco_square_ids, gray,
board)
-
+
if res2[1] is not None and res2[2] is not None and len(res2[1]) > 3:
charuco_corners = res2[1]
charuco_ids = res2[2]
-
+
return charuco_corners, charuco_ids, aruco_square_corners, aruco_square_ids
- def annotate_image_with_charuco_data(self, image, port_number, charuco_corners, charuco_ids)->bool:
+ def annotate_image_with_charuco_data(self, image, port_number, charuco_corners,
+ charuco_ids) -> bool:
full_charuco_detected_on_this_frame = False
if len(charuco_ids) == num_charuco_corners:
full_charuco_detected_on_this_frame = True
image_w_markers = cv2.aruco.drawDetectedCornersCharuco(image,
- np.array(charuco_corners),
- np.array(charuco_ids),
- (0,255,125,255)) #yellow? I think cv2 uses BGR instead of RGB?
+ np.array(charuco_corners),
+ np.array(charuco_ids),
+ (0, 255, 125, 255)) # yellow? I
+ # think cv2 uses BGR instead of RGB?
this_cam_name = "Camera " + str(port_number)
@@ -106,7 +100,8 @@ class BoardDetection:
len(charuco_ids)) + " of " + str(
num_charuco_corners) + " ChAruco Corner Points detected | Full Board Detected: " + str(
full_charuco_detected_on_this_frame)
- # TODO - Determine 'shared views' (i.e. frames in which a full board is detected by 2 cameras)
+ # TODO - Determine 'shared views' (i.e. frames in which a full board is detected by 2
+ # cameras)
# TODO - self.determine_shared_charuco_board_views()
# this_cam_shared_views_str = " | Shared Views: " + str(
# each_cameras_shared_board_view_count_total)
@@ -120,26 +115,29 @@ class BoardDetection:
cv2.FONT_HERSHEY_SIMPLEX, # font family
.5, # font size
(30, 10, 0, 255), # font color
- 2) # font stroke (draw a darker heavier font beneath a lighter/thinner copy for readability)
+ 2) # font stroke (draw a darker heavier font beneath a lighter/thinner copy for
+ # readability)
cv2.putText(
image_w_markers, # numpy array on which text is written
text_to_write_on_this_camera, # text
position, # position at which writing has to start
- cv2.FONT_HERSHEY_SIMPLEX, # font family (very limited selection, i think there's some interesting CV history here...)
+ cv2.FONT_HERSHEY_SIMPLEX,
+ # font family (very limited selection, i think there's some interesting CV history
+ # here...)
.5, # font size
(209, 180, 0, 255), # font color
- 1 ) # font stroke
+ 1) # font stroke
if full_charuco_detected_on_this_frame:
cv2.polylines(image_w_markers, np.int32([charuco_corners]), False, (0, 100, 255), 2)
# for these_corners in charuco_points_from_previous_frames:
# if len(these_corners)>0:
- # cv2.polylines(image_w_markers, np.int32([these_corners]), True, (0,100,255,255/2), 2)
+ # cv2.polylines(image_w_markers, np.int32([these_corners]), True, (0,100,255,
+ # 255/2), 2)
return True
-
def determine_shared_charuco_board_views(self):
pass
# # determine paired board views
@@ -148,14 +146,21 @@ class BoardDetection:
# for this_other_camera_num in range(multi_cam.num_cams):
# if this_other_camera_num != this_cam_num:
# if full_charuco_detected_on_this_frame[this_other_camera_num]:
- # each_cameras_shared_board_view_count_array[this_cam_num, this_other_camera_num] += 1
- # each_cameras_shared_board_view_count_total = np.sum(each_cameras_shared_board_view_count_array, axis=1)
+ # each_cameras_shared_board_view_count_array[this_cam_num,
+ # this_other_camera_num] += 1
+ # each_cameras_shared_board_view_count_total = np.sum(
+ # each_cameras_shared_board_view_count_array, axis=1)
# # print(each_cameras_shared_board_view_count)
#
# for this_pair_num, this_cam_pair in enumerate(camera_pair_list):
# camera_one_id = this_cam_pair[0]
# camera_two_id = this_cam_pair[1]
#
- # if full_charuco_detected_on_this_frame[camera_one_id] and full_charuco_detected_on_this_frame[
+ # if full_charuco_detected_on_this_frame[camera_one_id] and
+ # full_charuco_detected_on_this_frame[
# camera_two_id]:
- # camera_pair_joint_view_count[this_pair_num] += 1
\ No newline at end of file
+ # camera_pair_joint_view_count[this_pair_num] += 1
+
+
+if __name__ == "__main__":
+ asyncio.run(BoardDetection().process())
diff --git a/src/core_processor/processor.py b/src/core_processor/processor.py
index c64c4988..e0d23116 100644
--- a/src/core_processor/processor.py
+++ b/src/core_processor/processor.py
@@ -40,7 +40,7 @@ class FramePayload(NamedTuple):
class ImagePayload(NamedTuple):
- frames: List[FramePayload]
+ frames: List[FramePayload] = None
class CameraCaptureProcess:
@@ -68,7 +68,7 @@ class CameraCaptureProcess:
)
)
- _queue.put(
+ _queue.put_nowait(
ImagePayload(frames=image_list)
)
diff --git a/src/core_processor/testing/threading_single_cam.py b/src/core_processor/testing/threading_single_cam.py
new file mode 100644
index 00000000..39b3fbec
--- /dev/null
+++ b/src/core_processor/testing/threading_single_cam.py
@@ -0,0 +1,100 @@
+import time
+from threading import Thread
+
+import cv2
+
+
+class WebcamStream:
+ def __init__(self, stream_id=0):
+ self.stream_id = stream_id # default is 0 for primary camera
+
+ # opening video capture stream
+ self.vcap = cv2.VideoCapture(self.stream_id)
+ if self.vcap.isOpened() is False:
+ print("[Exiting]: Error accessing webcam stream.")
+ exit(0)
+ fps_input_stream = int(self.vcap.get(5))
+ print("FPS of webcam hardware/input stream: {}".format(fps_input_stream))
+
+ # reading a single frame from vcap stream for initializing
+ self.grabbed, self.frame = self.vcap.read()
+ if self.grabbed is False:
+ print('[Exiting] No more frames to read')
+ exit(0)
+
+ # self.stopped is set to False when frames are being read from self.vcap stream
+ self.stopped = True
+
+ # reference to the thread for reading next available frame from input stream
+ self.t = Thread(target=self.update, args=())
+ self.t.daemon = True # daemon threads keep running in the background while the program
+ # is executing
+
+ # method for starting the thread for grabbing next available frame in input stream
+ def start(self):
+ self.stopped = False
+ self.t.start()
+
+ # method for reading next frame
+
+ def update(self):
+ while True:
+ if self.stopped is True:
+ break
+ self.grabbed, self.frame = self.vcap.read()
+ if self.grabbed is False:
+ print('[Exiting] No more frames to read')
+ self.stopped = True
+ break
+ self.vcap.release()
+
+ # method for returning latest read frame
+ def read(self):
+ return self.frame
+
+ # method called to stop reading frames
+ def stop(self):
+ self.stopped = True
+
+ # initializing and starting multi-threaded webcam capture input stream
+
+
+webcam_stream1 = WebcamStream(stream_id=0) # stream_id = 0 is for primary camera
+webcam_stream2 = WebcamStream(stream_id=1) # stream_id = 0 is for primary camera
+webcam_stream1.start()
+webcam_stream2.start()
+
+# processing frames in input stream
+num_frames_processed = 0
+start = time.time()
+while True:
+ if webcam_stream1.stopped is True:
+ break
+ if webcam_stream2.stopped is True:
+ break
+ else:
+ frame1 = webcam_stream1.read()
+ frame2 = webcam_stream2.read()
+
+ # adding a delay for simulating time taken for processing a frame
+ delay = 0.03 # delay value in seconds. so, delay=1 is equivalent to 1 second
+ time.sleep(delay)
+ num_frames_processed += 1
+
+ cv2.imshow('frame1', frame1)
+ cv2.imshow('frame2', frame2)
+ key = cv2.waitKey(1)
+ if key == ord('q'):
+ break
+end = time.time()
+webcam_stream1.stop() # stop the webcam stream
+webcam_stream2.stop() # stop the webcam stream
+
+# printing time elapsed and fps
+elapsed = end - start
+fps = num_frames_processed / elapsed
+print(
+ "FPS: {} , Elapsed Time: {} , Frames Processed: {}".format(fps, elapsed, num_frames_processed))
+
+# closing all windows
+cv2.destroyAllWindows()