Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

now saves - need to fix board detetion next

Endurance Idehen915cf4c7feb5
diff --git a/src/api/middleware/cors.py b/src/api/middleware/cors.py
index adb00b8a..a19890ab 100644
--- a/src/api/middleware/cors.py
+++ b/src/api/middleware/cors.py
@@ -8,4 +8,4 @@ def cors(app):
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
- )
\ No newline at end of file
+ )
diff --git a/src/api/routes/__init__.py b/src/api/routes/__init__.py
index 70f150e6..808014f6 100644
--- a/src/api/routes/__init__.py
+++ b/src/api/routes/__init__.py
@@ -4,9 +4,4 @@ from src.api.routes.health.health_check_route import healthcheck_router
from src.api.routes.startup.startup import startup_router
# REGISTER NEW ROUTES HERE
-enabled_routers = [
- healthcheck_router,
- camera_router,
- cam_ws_router,
- startup_router
-]
+enabled_routers = [healthcheck_router, camera_router, cam_ws_router, startup_router]
diff --git a/src/api/routes/health/health_check_route.py b/src/api/routes/health/health_check_route.py
index 9a0b90d8..d4477efd 100644
--- a/src/api/routes/health/health_check_route.py
+++ b/src/api/routes/health/health_check_route.py
@@ -8,10 +8,9 @@ class HealthCheckResponse(BaseModel):
message: str = "OK"
-@healthcheck_router.get('/health')
+@healthcheck_router.get("/health")
def route():
try:
return HealthCheckResponse()
except:
raise ValueError("Unhealthy")
-
diff --git a/src/cameras/cv_camera_manager.py b/src/cameras/cv_camera_manager.py
index f2331834..d2fb5798 100644
--- a/src/cameras/cv_camera_manager.py
+++ b/src/cameras/cv_camera_manager.py
@@ -1,9 +1,14 @@
import logging
+import traceback
from contextlib import contextmanager
-from typing import List
+from typing import ContextManager, Dict, List
+
+import cv2
+from pydantic import BaseModel
from src.api.services.user_config import UserConfigService
from src.cameras.cam_singleton import get_or_create_cams
+from src.cameras.dtos.create_writer_options import CreateWriterOptions
from src.cameras.opencv_camera import OpenCVCamera
logger = logging.getLogger(__name__)
@@ -11,6 +16,14 @@ logger = logging.getLogger(__name__)
_cv_cams = None
+class CamAndWriterResponse(BaseModel):
+ cv_cam: OpenCVCamera
+ writer: cv2.VideoWriter
+
+ class Config:
+ arbitrary_types_allowed = True
+
+
class CVCameraManager:
def __init__(self):
self._config_service = UserConfigService()
@@ -44,45 +57,82 @@ class CVCameraManager:
return cv_cams
@contextmanager
- def start_capture_session(self, webcam_id: str = None):
+ def start_capture_session_single_cam(
+ self, webcam_id: str = None, create_writer_options: CreateWriterOptions = None
+ ) -> ContextManager[CamAndWriterResponse]:
"""
Context manager for easy start up, usage, and cleanup of camera resources.
Can capture frames from a single webcam or all webcams detected.
"""
- if webcam_id:
- self.start_frame_capture_on_cam_id(webcam_id)
- yield self.cv_cam_by_id(webcam_id)
- else:
- self.start_frame_capture_all_cams()
- yield self._cv_cams
-
- logger.debug("Cleaning up capture session")
- self.stop_frame_capture_all_cams()
+ writer = None
+ try:
+ writer = self._start_frame_capture_on_cam_id(
+ webcam_id, create_writer_options
+ )
+ cv_cam = self.cv_cam_by_id(webcam_id)
+ yield CamAndWriterResponse(cv_cam=cv_cam, writer=writer)
+ self._stop_frame_capture_all_cams()
+ except:
+ logger.error("Printing traceback from starting capture session by cam")
+ traceback.print_exc()
+ finally:
+ if writer:
+ writer.release()
+ logger.debug("Cleaning up capture session")
- def start_frame_capture_all_cams(self):
+ @contextmanager
+ def start_capture_session_all_cams(
+ self, create_writer_options: CreateWriterOptions = None
+ ) -> ContextManager[List[CamAndWriterResponse]]:
+ writer_dict = None
+ try:
+ writer_dict = self._start_frame_capture_all_cams(create_writer_options)
+ responses = [
+ CamAndWriterResponse(cv_cam=self.cv_cam_by_id(webcam_id), writer=writer)
+ for webcam_id, writer in writer_dict.items()
+ ]
+ yield responses
+ self._stop_frame_capture_all_cams()
+ except:
+ logger.error("Printing traceback from starting capture session by cam")
+ traceback.print_exc()
+ finally:
+ if writer_dict:
+ for writer in writer_dict.values():
+ writer.release()
+ logger.debug("Cleaning up capture session")
+
+ def _start_frame_capture_all_cams(
+ self, create_writer_options: CreateWriterOptions = None
+ ) -> Dict[str, cv2.VideoWriter]:
+ d = {}
for cv_cam in self._cv_cams:
cv_cam.connect()
cv_cam.start_frame_capture()
+ d[cv_cam.webcam_id_as_str] = cv_cam.create_video_writer(
+ create_writer_options
+ )
+
+ return d
- def start_frame_capture_on_cam_id(self, webcam_id: str):
+ def _start_frame_capture_on_cam_id(
+ self, webcam_id: str, create_writer_options: CreateWriterOptions = None
+ ) -> cv2.VideoWriter:
filtered_cams = list(
filter(lambda c: c.webcam_id_as_str == str(webcam_id), self._cv_cams)
)
assert (
len(filtered_cams) == 1
- ), "THe CV Cams list should only have 1 cam per webcam_id"
+ ), "The CV Cams list should only have 1 cam per webcam_id"
cv_cam = filtered_cams[0]
cv_cam.connect()
cv_cam.start_frame_capture()
+ return cv_cam.create_video_writer(create_writer_options)
- def stop_frame_capture_all_cams(self):
+ def _stop_frame_capture_all_cams(self):
for cv_cam in self._cv_cams:
cv_cam.stop_frame_capture()
- def close_all_cameras(self):
- close_all_cameras(self._cv_cams)
-
-
-def close_all_cameras(cv_cams: List[OpenCVCamera]):
- for cv_cam in cv_cams:
- cv_cam.close()
+ def _close_all_cameras(self):
+ for cv_cam in self._cv_cams:
+ cv_cam.close()
diff --git a/src/cameras/dtos/create_writer_options.py b/src/cameras/dtos/create_writer_options.py
new file mode 100644
index 00000000..66977797
--- /dev/null
+++ b/src/cameras/dtos/create_writer_options.py
@@ -0,0 +1,20 @@
+import time
+from pathlib import Path
+
+from pydantic import BaseModel
+
+
+class CreateWriterOptions(BaseModel):
+ filename_and_ext: str
+ parent_folder: str
+
+
+def get_canonical_time_str():
+ return time.strftime("%Y%m%d_%H%M%S")
+
+
+def get_base_options():
+ return CreateWriterOptions(
+ filename_and_ext=f"{get_canonical_time_str()}.avi",
+ parent_folder="raw_frame_capture",
+ )
diff --git a/src/cameras/frame_grabber.py b/src/cameras/frame_grabber.py
index a30fb20f..20f7f394 100644
--- a/src/cameras/frame_grabber.py
+++ b/src/cameras/frame_grabber.py
@@ -1,5 +1,4 @@
import logging
-import math
import threading
import time
import traceback
diff --git a/src/cameras/open_cv_cam_writer.py b/src/cameras/open_cv_cam_writer.py
index b1fe01b1..a6ae676c 100644
--- a/src/cameras/open_cv_cam_writer.py
+++ b/src/cameras/open_cv_cam_writer.py
@@ -1,20 +1,42 @@
+import logging
import os
-import time
from pathlib import Path
import cv2
+from src.cameras.dtos.create_writer_options import CreateWriterOptions, get_base_options
+
+logger = logging.getLogger(__name__)
+
class OpenCVCamWriter:
- def create_writer(self, video_capture_obj):
+ def create_writer(self, video_capture_obj, options: CreateWriterOptions = None):
+ """
+ Create writer makes a writer based off a video capture object
+ If a write location is specified, then the videos are written there.
+ If not, the videos are written to the current working directory
+
+ OS specifics are handled by the Path module.
+ """
+
+ if options is None:
+ options = get_base_options()
frame_width = int(video_capture_obj.get(3))
frame_height = int(video_capture_obj.get(4))
fps = int(video_capture_obj.get(5))
- timestr = time.strftime("%Y%m%d_%H%M%S")
- p = Path().joinpath(os.getcwd(), f"{timestr}.avi").resolve()
+
+ final_path = (
+ Path()
+ .joinpath(os.getcwd(), options.parent_folder, options.filename_and_ext)
+ .resolve()
+ )
+ logger.debug(f"Video save path: {final_path}")
return cv2.VideoWriter(
- "outfile.avi",
- cv2.VideoWriter_fourcc("M", "J", "P", "G"),
+ str(final_path),
+ cv2.VideoWriter_fourcc(*"MJPG"),
fps,
(frame_width, frame_height),
)
+
+ def close_writer(self, writer: cv2.VideoWriter):
+ writer.release()
diff --git a/src/cameras/opencv_camera.py b/src/cameras/opencv_camera.py
index 661d9190..b20bba98 100644
--- a/src/cameras/opencv_camera.py
+++ b/src/cameras/opencv_camera.py
@@ -1,23 +1,31 @@
import logging
import platform
import time
+import traceback
+from pathlib import Path
import cv2
from pydantic import BaseModel
from src.cameras.dto import FramePayload
from src.cameras.frame_grabber import FrameThread
-from src.cameras.open_cv_cam_writer import OpenCVCamWriter
+from src.cameras.open_cv_cam_writer import CreateWriterOptions, OpenCVCamWriter
logger = logging.getLogger(__name__)
+def _get_home_dir():
+ return str(Path.home())
+
+
class WebcamConfig(BaseModel):
webcam_id: int = 0
exposure: int = -6
resolution_width: int = 800
resolution_height: int = 600
save_video: bool = True
+ fourcc: str = "MJPG"
+ base_save_video_dir = _get_home_dir()
class OpenCVCamera:
@@ -27,10 +35,12 @@ class OpenCVCamera:
_opencv_video_capture_object: cv2.VideoCapture = None
_running_thread: FrameThread = None
+ _writer: OpenCVCamWriter
def __init__(self, config: WebcamConfig):
self._config = config
self._name = f"Camera {self._config.webcam_id}"
+ self._writer = OpenCVCamWriter()
@property
def webcam_id_as_str(self):
@@ -52,6 +62,10 @@ class OpenCVCamera:
def current_fps_short(self) -> str:
return "{:.2f}".format(self._running_thread.current_fps)
+ @property
+ def latest_frame(self):
+ return self._running_thread.latest_frame
+
def connect(self):
if platform.system() == "Windows":
cap_backend = cv2.CAP_DSHOW
@@ -61,6 +75,7 @@ class OpenCVCamera:
self._opencv_video_capture_object = cv2.VideoCapture(
self._config.webcam_id, cap_backend
)
+ self._apply_configuration()
success, image = self._opencv_video_capture_object.read()
if not success:
@@ -70,26 +85,16 @@ class OpenCVCamera:
)
)
return success
-
- # set camera stream parameters
- # self.opencv_video_capture_object.set(cv2.CAP_PROP_EXPOSURE, self._config.exposure)
- # self.opencv_video_capture_object.set(
- # cv2.CAP_PROP_FRAME_WIDTH, self._config.resolution_width
- # )
- # self.opencv_video_capture_object.set(
- # cv2.CAP_PROP_FRAME_HEIGHT, self._config.resolution_height
- # )
-
- self._opencv_video_capture_object.set(
- cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc("M", "J", "P", "G")
+ logger.debug(f"Camera found at port number {self._config.webcam_id}")
+ fps_input_stream = int(self._opencv_video_capture_object.get(5))
+ logger.debug("FPS of webcam hardware/input stream: {}".format(fps_input_stream))
+ return success
+
+ def create_video_writer(self, create_writer_options: CreateWriterOptions):
+ return self._writer.create_writer(
+ self._opencv_video_capture_object, options=create_writer_options
)
- if success:
- logger.debug(f"Camera found at port number {self._config.webcam_id}")
- fps_input_stream = int(self._opencv_video_capture_object.get(5))
- print("FPS of webcam hardware/input stream: {}".format(fps_input_stream))
- return success
-
def start_frame_capture(self):
if self.is_capturing_frames:
logger.debug(
@@ -105,37 +110,52 @@ class OpenCVCamera:
def _create_thread(self):
return FrameThread(
get_next_frame=self.get_next_frame,
- writer=OpenCVCamWriter().create_writer(self._opencv_video_capture_object),
+ writer=self._writer.create_writer(self._opencv_video_capture_object),
save_video=self._config.save_video,
)
- @property
- def latest_frame(self):
- return self._running_thread.latest_frame
+ def _apply_configuration(self):
+ # set camera stream parameters
+ self._opencv_video_capture_object.set(
+ cv2.CAP_PROP_EXPOSURE, self._config.exposure
+ )
+ self._opencv_video_capture_object.set(
+ cv2.CAP_PROP_FRAME_WIDTH, self._config.resolution_width
+ )
+ self._opencv_video_capture_object.set(
+ cv2.CAP_PROP_FRAME_HEIGHT, self._config.resolution_height
+ )
+ self._opencv_video_capture_object.set(
+ cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*self._config.fourcc)
+ )
def get_next_frame(self):
timestamp_ns_pre_grab = time.time_ns()
# Why grab not read? see ->
# https://stackoverflow.com/questions/57716962/difference-between-video-capture-read-and
# -grab
- grab_success = self._opencv_video_capture_object.grab()
+ if not self._opencv_video_capture_object.grab():
+ return FramePayload(False, None, None)
+
timestamp_ns_post_grab = time.time_ns()
timestamp_ns = (timestamp_ns_pre_grab + timestamp_ns_post_grab) / 2
- if grab_success:
- success, image = self._opencv_video_capture_object.retrieve()
- return FramePayload(success, image, timestamp_ns)
-
- return FramePayload(False, None, None)
+ success, image = self._opencv_video_capture_object.retrieve()
+ return FramePayload(success, image, timestamp_ns)
def stop_frame_capture(self):
self.close()
def close(self):
- self._running_thread.stop()
- while self._running_thread.is_alive():
- # wait for thread to die.
- # TODO: use threading.Event for synchronize mainthread vs other threads
- time.sleep(0.1)
- self._opencv_video_capture_object.release()
- logger.info("Closed {}".format(self._name))
+ try:
+ self._running_thread.stop()
+ while self._running_thread.is_alive():
+ # wait for thread to die.
+ # TODO: use threading.Event for synchronize mainthread vs other threads
+ time.sleep(0.1)
+ except:
+ logger.error("Printing traceback")
+ traceback.print_exc()
+ finally:
+ logger.info("Closed {}".format(self._name))
+ self._opencv_video_capture_object.release()
diff --git a/src/core_processor/board_detection/base_pose_estimation.py b/src/core_processor/board_detection/base_pose_estimation.py
index 15b6d9a3..ea9063b4 100644
--- a/src/core_processor/board_detection/base_pose_estimation.py
+++ b/src/core_processor/board_detection/base_pose_estimation.py
@@ -16,18 +16,19 @@ def detect_charuco_board(image):
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 0.00001)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
- aruco_square_corners, aruco_square_ids, rejectedImgPoints = cv2.aruco.detectMarkers(gray,
- aruco_dict)
+ aruco_square_corners, aruco_square_ids, rejectedImgPoints = cv2.aruco.detectMarkers(
+ gray, aruco_dict
+ )
if len(aruco_square_corners) > 0:
# SUB PIXEL DETECTION
for this_corner in aruco_square_corners:
- cv2.cornerSubPix(gray, this_corner,
- winSize=(3, 3),
- zeroZone=(-1, -1),
- criteria=criteria)
- res2 = cv2.aruco.interpolateCornersCharuco(aruco_square_corners, aruco_square_ids, gray,
- board)
+ cv2.cornerSubPix(
+ gray, this_corner, winSize=(3, 3), zeroZone=(-1, -1), criteria=criteria
+ )
+ res2 = cv2.aruco.interpolateCornersCharuco(
+ aruco_square_corners, aruco_square_ids, gray, board
+ )
if res2[1] is not None and res2[2] is not None and len(res2[1]) > 3:
charuco_corners = res2[1]
diff --git a/src/core_processor/board_detection/board_detection.py b/src/core_processor/board_detection/board_detection.py
index 56f53404..fadb7366 100644
--- a/src/core_processor/board_detection/board_detection.py
+++ b/src/core_processor/board_detection/board_detection.py
@@ -19,7 +19,10 @@ class BoardDetection:
self._cam_manager = cam_manager
async def process_by_cam_id(self, webcam_id: str, cb):
- with self._cam_manager.start_capture_session(webcam_id) as cv_cam:
+ with self._cam_manager.start_capture_session_single_cam(
+ webcam_id, save_video=True
+ ) as cam_response:
+ cv_cam = cam_response.cv_cam
try:
while True:
if not cv_cam.is_capturing_frames:
@@ -36,26 +39,30 @@ class BoardDetection:
Opens Camera using OpenCV and begins image processing for charuco board
If return images is true, the images are returned to the caller
"""
- cam_manager: CVCameraManager = CVCameraManager()
- cv_cams = cam_manager.cv_cams
-
# if its already capturing frames, this is a no-op
- with cam_manager.start_capture_session():
+
+ with self._cam_manager.start_capture_session_all_cams(
+ save_video=True
+ ) as session_obj:
try:
while True:
exit_key = cv2.waitKey(1)
if exit_key == 27:
logger.info("ESC has been pressed.")
break
- for cv_cam in cv_cams:
+ for response in session_obj:
+ cv_cam = response.cv_cam
+ writer = response.writer
frame = self._process_single_cam_frame(cv_cam)
if frame is not None:
+ writer.write(frame)
cv2.imshow(cv_cam.webcam_id_as_str, frame)
except:
logger.error("Printing traceback")
traceback.print_exc()
finally:
- for cv_cam in cv_cams:
+ for response in session_obj:
+ cv_cam = response.cv_cam
logger.info(f"Destroy window {cv_cam.webcam_id_as_str}")
cv2.destroyWindow(cv_cam.webcam_id_as_str)
cv2.waitKey(1)
diff --git a/src/core_processor/board_detection/charuco_constants.py b/src/core_processor/board_detection/charuco_constants.py
index a3e71562..3816642c 100644
--- a/src/core_processor/board_detection/charuco_constants.py
+++ b/src/core_processor/board_detection/charuco_constants.py
@@ -4,6 +4,5 @@ aruco_dict = cv2.aruco.Dictionary_get(cv2.aruco.DICT_4X4_250)
charuco_length = 7
charuco_width = 5
-board = cv2.aruco.CharucoBoard_create(charuco_length, charuco_width, 1, .8, aruco_dict)
+board = cv2.aruco.CharucoBoard_create(charuco_length, charuco_width, 1, 0.8, aruco_dict)
num_charuco_corners = (charuco_length - 1) * (charuco_width - 1)
-
diff --git a/src/core_processor/board_detection/shared_board_view_detection.py b/src/core_processor/board_detection/shared_board_view_detection.py
index 173af06f..0d323672 100644
--- a/src/core_processor/board_detection/shared_board_view_detection.py
+++ b/src/core_processor/board_detection/shared_board_view_detection.py
@@ -1,4 +1,3 @@
-
def determine_shared_charuco_board_views(self):
pass
# # determine paired board views
@@ -20,4 +19,4 @@ def determine_shared_charuco_board_views(self):
# if full_charuco_detected_on_this_frame[camera_one_id] and
# full_charuco_detected_on_this_frame[
# camera_two_id]:
- # camera_pair_joint_view_count[this_pair_num] += 1
\ No newline at end of file
+ # camera_pair_joint_view_count[this_pair_num] += 1
diff --git a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py
index 03ddc484..31b13f7c 100644
--- a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py
+++ b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py
@@ -1,10 +1,17 @@
import logging
+import os
import traceback
+from pathlib import Path
+from uuid import uuid4
import cv2
import mediapipe as mp
from src.cameras.cv_camera_manager import CVCameraManager
+from src.cameras.dtos.create_writer_options import (
+ CreateWriterOptions,
+ get_canonical_time_str,
+)
from src.core_processor.utils.image_fps_writer import write_fps_to_image
mp_holistic = mp.solutions.holistic
@@ -19,7 +26,14 @@ class MediapipeSkeletonDetection:
self._cam_manager = cam_manager
async def process_as_frame_loop(self, webcam_id, cb, model_complexity: int = 1):
- with self._cam_manager.start_capture_session(webcam_id=webcam_id) as cv_cam:
+ timestr = get_canonical_time_str()
+ options = CreateWriterOptions(
+ filename_and_ext=f"{uuid4()}.{timestr}.avi",
+ relative_folder_path=Path().joinpath("skeleton_detection"),
+ )
+ with self._cam_manager.start_capture_session_single_cam(
+ webcam_id=webcam_id, create_writer_options=options
+ ) as session_obj:
with mp_holistic.Holistic(
min_detection_confidence=0.5,
min_tracking_confidence=0.5,
@@ -27,7 +41,10 @@ class MediapipeSkeletonDetection:
) as holistic:
try:
while True:
- image = self._process_single_cam_frame(holistic, cv_cam)
+ image = self._process_single_cam_frame(
+ holistic, session_obj.cv_cam
+ )
+ session_obj.writer.write(image)
if cb and image is not None:
await cb(image)
except Exception as e:
@@ -36,7 +53,12 @@ class MediapipeSkeletonDetection:
raise e
def process(self, model_complexity: int = 1):
- with self._cam_manager.start_capture_session() as cv_cams:
+ timestr = get_canonical_time_str()
+ options = CreateWriterOptions(
+ filename_and_ext=f"{uuid4()}.{timestr}.avi",
+ parent_folder="skeleton_detection",
+ )
+ with self._cam_manager.start_capture_session_all_cams(options) as session_obj:
with mp_holistic.Holistic(
min_detection_confidence=0.5,
min_tracking_confidence=0.5,
@@ -48,14 +70,24 @@ class MediapipeSkeletonDetection:
if exit_key == 27:
logger.info("ESC has been pressed.")
break
- for cv_cam in cv_cams:
+ for response in session_obj:
+ cv_cam = response.cv_cam
+ writer = response.writer
image = self._process_single_cam_frame(holistic, cv_cam)
- cv2.imshow(cv_cam.webcam_id_as_str, image)
+ writer.write(image)
+ try:
+ cv2.imshow(cv_cam.webcam_id_as_str, image)
+ except:
+ logger.debug(
+ "Failed to draw image for some reason for cv2.imshow"
+ )
+ traceback.print_exc()
except:
logger.error("Printing traceback")
traceback.print_exc()
finally:
- for cv_cam in cv_cams:
+ for response in session_obj:
+ cv_cam = response.cv_cam
logger.info(f"Destroy window {cv_cam.webcam_id_as_str}")
cv2.destroyWindow(cv_cam.webcam_id_as_str)
cv2.waitKey(1)