Free Motion Capture for Everyone πβ¨
TypeScript
now saves - need to fix board detetion next
Endurance Idehen915cf4c7feb5
diff --git a/src/api/middleware/cors.py b/src/api/middleware/cors.pyindex adb00b8a..a19890ab 100644--- a/src/api/middleware/cors.py+++ b/src/api/middleware/cors.py@@ -8,4 +8,4 @@ def cors(app):allow_credentials=True,allow_methods=["*"],allow_headers=["*"],- )\ No newline at end of file+ )diff --git a/src/api/routes/__init__.py b/src/api/routes/__init__.pyindex 70f150e6..808014f6 100644--- a/src/api/routes/__init__.py+++ b/src/api/routes/__init__.py@@ -4,9 +4,4 @@ from src.api.routes.health.health_check_route import healthcheck_routerfrom src.api.routes.startup.startup import startup_router# REGISTER NEW ROUTES HERE-enabled_routers = [- healthcheck_router,- camera_router,- cam_ws_router,- startup_router-]+enabled_routers = [healthcheck_router, camera_router, cam_ws_router, startup_router]diff --git a/src/api/routes/health/health_check_route.py b/src/api/routes/health/health_check_route.pyindex 9a0b90d8..d4477efd 100644--- a/src/api/routes/health/health_check_route.py+++ b/src/api/routes/health/health_check_route.py@@ -8,10 +8,9 @@ class HealthCheckResponse(BaseModel):message: str = "OK"-@healthcheck_router.get('/health')+@healthcheck_router.get("/health")def route():try:return HealthCheckResponse()except:raise ValueError("Unhealthy")-diff --git a/src/cameras/cv_camera_manager.py b/src/cameras/cv_camera_manager.pyindex f2331834..d2fb5798 100644--- a/src/cameras/cv_camera_manager.py+++ b/src/cameras/cv_camera_manager.py@@ -1,9 +1,14 @@import logging+import tracebackfrom contextlib import contextmanager-from typing import List+from typing import ContextManager, Dict, List++import cv2+from pydantic import BaseModelfrom src.api.services.user_config import UserConfigServicefrom src.cameras.cam_singleton import get_or_create_cams+from src.cameras.dtos.create_writer_options import CreateWriterOptionsfrom src.cameras.opencv_camera import OpenCVCameralogger = logging.getLogger(__name__)@@ -11,6 +16,14 @@ logger = logging.getLogger(__name__)_cv_cams = None+class CamAndWriterResponse(BaseModel):+ cv_cam: OpenCVCamera+ writer: cv2.VideoWriter++ class Config:+ arbitrary_types_allowed = True++class CVCameraManager:def __init__(self):self._config_service = UserConfigService()@@ -44,45 +57,82 @@ class CVCameraManager:return cv_cams@contextmanager- def start_capture_session(self, webcam_id: str = None):+ def start_capture_session_single_cam(+ self, webcam_id: str = None, create_writer_options: CreateWriterOptions = None+ ) -> ContextManager[CamAndWriterResponse]:"""Context manager for easy start up, usage, and cleanup of camera resources.Can capture frames from a single webcam or all webcams detected."""- if webcam_id:- self.start_frame_capture_on_cam_id(webcam_id)- yield self.cv_cam_by_id(webcam_id)- else:- self.start_frame_capture_all_cams()- yield self._cv_cams-- logger.debug("Cleaning up capture session")- self.stop_frame_capture_all_cams()+ writer = None+ try:+ writer = self._start_frame_capture_on_cam_id(+ webcam_id, create_writer_options+ )+ cv_cam = self.cv_cam_by_id(webcam_id)+ yield CamAndWriterResponse(cv_cam=cv_cam, writer=writer)+ self._stop_frame_capture_all_cams()+ except:+ logger.error("Printing traceback from starting capture session by cam")+ traceback.print_exc()+ finally:+ if writer:+ writer.release()+ logger.debug("Cleaning up capture session")- def start_frame_capture_all_cams(self):+ @contextmanager+ def start_capture_session_all_cams(+ self, create_writer_options: CreateWriterOptions = None+ ) -> ContextManager[List[CamAndWriterResponse]]:+ writer_dict = None+ try:+ writer_dict = self._start_frame_capture_all_cams(create_writer_options)+ responses = [+ CamAndWriterResponse(cv_cam=self.cv_cam_by_id(webcam_id), writer=writer)+ for webcam_id, writer in writer_dict.items()+ ]+ yield responses+ self._stop_frame_capture_all_cams()+ except:+ logger.error("Printing traceback from starting capture session by cam")+ traceback.print_exc()+ finally:+ if writer_dict:+ for writer in writer_dict.values():+ writer.release()+ logger.debug("Cleaning up capture session")++ def _start_frame_capture_all_cams(+ self, create_writer_options: CreateWriterOptions = None+ ) -> Dict[str, cv2.VideoWriter]:+ d = {}for cv_cam in self._cv_cams:cv_cam.connect()cv_cam.start_frame_capture()+ d[cv_cam.webcam_id_as_str] = cv_cam.create_video_writer(+ create_writer_options+ )++ return d- def start_frame_capture_on_cam_id(self, webcam_id: str):+ def _start_frame_capture_on_cam_id(+ self, webcam_id: str, create_writer_options: CreateWriterOptions = None+ ) -> cv2.VideoWriter:filtered_cams = list(filter(lambda c: c.webcam_id_as_str == str(webcam_id), self._cv_cams))assert (len(filtered_cams) == 1- ), "THe CV Cams list should only have 1 cam per webcam_id"+ ), "The CV Cams list should only have 1 cam per webcam_id"cv_cam = filtered_cams[0]cv_cam.connect()cv_cam.start_frame_capture()+ return cv_cam.create_video_writer(create_writer_options)- def stop_frame_capture_all_cams(self):+ def _stop_frame_capture_all_cams(self):for cv_cam in self._cv_cams:cv_cam.stop_frame_capture()- def close_all_cameras(self):- close_all_cameras(self._cv_cams)---def close_all_cameras(cv_cams: List[OpenCVCamera]):- for cv_cam in cv_cams:- cv_cam.close()+ def _close_all_cameras(self):+ for cv_cam in self._cv_cams:+ cv_cam.close()diff --git a/src/cameras/dtos/create_writer_options.py b/src/cameras/dtos/create_writer_options.pynew file mode 100644index 00000000..66977797--- /dev/null+++ b/src/cameras/dtos/create_writer_options.py@@ -0,0 +1,20 @@+import time+from pathlib import Path++from pydantic import BaseModel+++class CreateWriterOptions(BaseModel):+ filename_and_ext: str+ parent_folder: str+++def get_canonical_time_str():+ return time.strftime("%Y%m%d_%H%M%S")+++def get_base_options():+ return CreateWriterOptions(+ filename_and_ext=f"{get_canonical_time_str()}.avi",+ parent_folder="raw_frame_capture",+ )diff --git a/src/cameras/frame_grabber.py b/src/cameras/frame_grabber.pyindex a30fb20f..20f7f394 100644--- a/src/cameras/frame_grabber.py+++ b/src/cameras/frame_grabber.py@@ -1,5 +1,4 @@import logging-import mathimport threadingimport timeimport tracebackdiff --git a/src/cameras/open_cv_cam_writer.py b/src/cameras/open_cv_cam_writer.pyindex b1fe01b1..a6ae676c 100644--- a/src/cameras/open_cv_cam_writer.py+++ b/src/cameras/open_cv_cam_writer.py@@ -1,20 +1,42 @@+import loggingimport os-import timefrom pathlib import Pathimport cv2+from src.cameras.dtos.create_writer_options import CreateWriterOptions, get_base_options++logger = logging.getLogger(__name__)+class OpenCVCamWriter:- def create_writer(self, video_capture_obj):+ def create_writer(self, video_capture_obj, options: CreateWriterOptions = None):+ """+ Create writer makes a writer based off a video capture object+ If a write location is specified, then the videos are written there.+ If not, the videos are written to the current working directory++ OS specifics are handled by the Path module.+ """++ if options is None:+ options = get_base_options()frame_width = int(video_capture_obj.get(3))frame_height = int(video_capture_obj.get(4))fps = int(video_capture_obj.get(5))- timestr = time.strftime("%Y%m%d_%H%M%S")- p = Path().joinpath(os.getcwd(), f"{timestr}.avi").resolve()++ final_path = (+ Path()+ .joinpath(os.getcwd(), options.parent_folder, options.filename_and_ext)+ .resolve()+ )+ logger.debug(f"Video save path: {final_path}")return cv2.VideoWriter(- "outfile.avi",- cv2.VideoWriter_fourcc("M", "J", "P", "G"),+ str(final_path),+ cv2.VideoWriter_fourcc(*"MJPG"),fps,(frame_width, frame_height),)++ def close_writer(self, writer: cv2.VideoWriter):+ writer.release()diff --git a/src/cameras/opencv_camera.py b/src/cameras/opencv_camera.pyindex 661d9190..b20bba98 100644--- a/src/cameras/opencv_camera.py+++ b/src/cameras/opencv_camera.py@@ -1,23 +1,31 @@import loggingimport platformimport time+import traceback+from pathlib import Pathimport cv2from pydantic import BaseModelfrom src.cameras.dto import FramePayloadfrom src.cameras.frame_grabber import FrameThread-from src.cameras.open_cv_cam_writer import OpenCVCamWriter+from src.cameras.open_cv_cam_writer import CreateWriterOptions, OpenCVCamWriterlogger = logging.getLogger(__name__)+def _get_home_dir():+ return str(Path.home())++class WebcamConfig(BaseModel):webcam_id: int = 0exposure: int = -6resolution_width: int = 800resolution_height: int = 600save_video: bool = True+ fourcc: str = "MJPG"+ base_save_video_dir = _get_home_dir()class OpenCVCamera:@@ -27,10 +35,12 @@ class OpenCVCamera:_opencv_video_capture_object: cv2.VideoCapture = None_running_thread: FrameThread = None+ _writer: OpenCVCamWriterdef __init__(self, config: WebcamConfig):self._config = configself._name = f"Camera {self._config.webcam_id}"+ self._writer = OpenCVCamWriter()@propertydef webcam_id_as_str(self):@@ -52,6 +62,10 @@ class OpenCVCamera:def current_fps_short(self) -> str:return "{:.2f}".format(self._running_thread.current_fps)+ @property+ def latest_frame(self):+ return self._running_thread.latest_frame+def connect(self):if platform.system() == "Windows":cap_backend = cv2.CAP_DSHOW@@ -61,6 +75,7 @@ class OpenCVCamera:self._opencv_video_capture_object = cv2.VideoCapture(self._config.webcam_id, cap_backend)+ self._apply_configuration()success, image = self._opencv_video_capture_object.read()if not success:@@ -70,26 +85,16 @@ class OpenCVCamera:))return success-- # set camera stream parameters- # self.opencv_video_capture_object.set(cv2.CAP_PROP_EXPOSURE, self._config.exposure)- # self.opencv_video_capture_object.set(- # cv2.CAP_PROP_FRAME_WIDTH, self._config.resolution_width- # )- # self.opencv_video_capture_object.set(- # cv2.CAP_PROP_FRAME_HEIGHT, self._config.resolution_height- # )-- self._opencv_video_capture_object.set(- cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc("M", "J", "P", "G")+ logger.debug(f"Camera found at port number {self._config.webcam_id}")+ fps_input_stream = int(self._opencv_video_capture_object.get(5))+ logger.debug("FPS of webcam hardware/input stream: {}".format(fps_input_stream))+ return success++ def create_video_writer(self, create_writer_options: CreateWriterOptions):+ return self._writer.create_writer(+ self._opencv_video_capture_object, options=create_writer_options)- if success:- logger.debug(f"Camera found at port number {self._config.webcam_id}")- fps_input_stream = int(self._opencv_video_capture_object.get(5))- print("FPS of webcam hardware/input stream: {}".format(fps_input_stream))- return success-def start_frame_capture(self):if self.is_capturing_frames:logger.debug(@@ -105,37 +110,52 @@ class OpenCVCamera:def _create_thread(self):return FrameThread(get_next_frame=self.get_next_frame,- writer=OpenCVCamWriter().create_writer(self._opencv_video_capture_object),+ writer=self._writer.create_writer(self._opencv_video_capture_object),save_video=self._config.save_video,)- @property- def latest_frame(self):- return self._running_thread.latest_frame+ def _apply_configuration(self):+ # set camera stream parameters+ self._opencv_video_capture_object.set(+ cv2.CAP_PROP_EXPOSURE, self._config.exposure+ )+ self._opencv_video_capture_object.set(+ cv2.CAP_PROP_FRAME_WIDTH, self._config.resolution_width+ )+ self._opencv_video_capture_object.set(+ cv2.CAP_PROP_FRAME_HEIGHT, self._config.resolution_height+ )+ self._opencv_video_capture_object.set(+ cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*self._config.fourcc)+ )def get_next_frame(self):timestamp_ns_pre_grab = time.time_ns()# Why grab not read? see -># https://stackoverflow.com/questions/57716962/difference-between-video-capture-read-and# -grab- grab_success = self._opencv_video_capture_object.grab()+ if not self._opencv_video_capture_object.grab():+ return FramePayload(False, None, None)+timestamp_ns_post_grab = time.time_ns()timestamp_ns = (timestamp_ns_pre_grab + timestamp_ns_post_grab) / 2- if grab_success:- success, image = self._opencv_video_capture_object.retrieve()- return FramePayload(success, image, timestamp_ns)-- return FramePayload(False, None, None)+ success, image = self._opencv_video_capture_object.retrieve()+ return FramePayload(success, image, timestamp_ns)def stop_frame_capture(self):self.close()def close(self):- self._running_thread.stop()- while self._running_thread.is_alive():- # wait for thread to die.- # TODO: use threading.Event for synchronize mainthread vs other threads- time.sleep(0.1)- self._opencv_video_capture_object.release()- logger.info("Closed {}".format(self._name))+ try:+ self._running_thread.stop()+ while self._running_thread.is_alive():+ # wait for thread to die.+ # TODO: use threading.Event for synchronize mainthread vs other threads+ time.sleep(0.1)+ except:+ logger.error("Printing traceback")+ traceback.print_exc()+ finally:+ logger.info("Closed {}".format(self._name))+ self._opencv_video_capture_object.release()diff --git a/src/core_processor/board_detection/base_pose_estimation.py b/src/core_processor/board_detection/base_pose_estimation.pyindex 15b6d9a3..ea9063b4 100644--- a/src/core_processor/board_detection/base_pose_estimation.py+++ b/src/core_processor/board_detection/base_pose_estimation.py@@ -16,18 +16,19 @@ def detect_charuco_board(image):criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 0.00001)gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)- aruco_square_corners, aruco_square_ids, rejectedImgPoints = cv2.aruco.detectMarkers(gray,- aruco_dict)+ aruco_square_corners, aruco_square_ids, rejectedImgPoints = cv2.aruco.detectMarkers(+ gray, aruco_dict+ )if len(aruco_square_corners) > 0:# SUB PIXEL DETECTIONfor this_corner in aruco_square_corners:- cv2.cornerSubPix(gray, this_corner,- winSize=(3, 3),- zeroZone=(-1, -1),- criteria=criteria)- res2 = cv2.aruco.interpolateCornersCharuco(aruco_square_corners, aruco_square_ids, gray,- board)+ cv2.cornerSubPix(+ gray, this_corner, winSize=(3, 3), zeroZone=(-1, -1), criteria=criteria+ )+ res2 = cv2.aruco.interpolateCornersCharuco(+ aruco_square_corners, aruco_square_ids, gray, board+ )if res2[1] is not None and res2[2] is not None and len(res2[1]) > 3:charuco_corners = res2[1]diff --git a/src/core_processor/board_detection/board_detection.py b/src/core_processor/board_detection/board_detection.pyindex 56f53404..fadb7366 100644--- a/src/core_processor/board_detection/board_detection.py+++ b/src/core_processor/board_detection/board_detection.py@@ -19,7 +19,10 @@ class BoardDetection:self._cam_manager = cam_managerasync def process_by_cam_id(self, webcam_id: str, cb):- with self._cam_manager.start_capture_session(webcam_id) as cv_cam:+ with self._cam_manager.start_capture_session_single_cam(+ webcam_id, save_video=True+ ) as cam_response:+ cv_cam = cam_response.cv_camtry:while True:if not cv_cam.is_capturing_frames:@@ -36,26 +39,30 @@ class BoardDetection:Opens Camera using OpenCV and begins image processing for charuco boardIf return images is true, the images are returned to the caller"""- cam_manager: CVCameraManager = CVCameraManager()- cv_cams = cam_manager.cv_cams-# if its already capturing frames, this is a no-op- with cam_manager.start_capture_session():++ with self._cam_manager.start_capture_session_all_cams(+ save_video=True+ ) as session_obj:try:while True:exit_key = cv2.waitKey(1)if exit_key == 27:logger.info("ESC has been pressed.")break- for cv_cam in cv_cams:+ for response in session_obj:+ cv_cam = response.cv_cam+ writer = response.writerframe = self._process_single_cam_frame(cv_cam)if frame is not None:+ writer.write(frame)cv2.imshow(cv_cam.webcam_id_as_str, frame)except:logger.error("Printing traceback")traceback.print_exc()finally:- for cv_cam in cv_cams:+ for response in session_obj:+ cv_cam = response.cv_camlogger.info(f"Destroy window {cv_cam.webcam_id_as_str}")cv2.destroyWindow(cv_cam.webcam_id_as_str)cv2.waitKey(1)diff --git a/src/core_processor/board_detection/charuco_constants.py b/src/core_processor/board_detection/charuco_constants.pyindex a3e71562..3816642c 100644--- a/src/core_processor/board_detection/charuco_constants.py+++ b/src/core_processor/board_detection/charuco_constants.py@@ -4,6 +4,5 @@ aruco_dict = cv2.aruco.Dictionary_get(cv2.aruco.DICT_4X4_250)charuco_length = 7charuco_width = 5-board = cv2.aruco.CharucoBoard_create(charuco_length, charuco_width, 1, .8, aruco_dict)+board = cv2.aruco.CharucoBoard_create(charuco_length, charuco_width, 1, 0.8, aruco_dict)num_charuco_corners = (charuco_length - 1) * (charuco_width - 1)-diff --git a/src/core_processor/board_detection/shared_board_view_detection.py b/src/core_processor/board_detection/shared_board_view_detection.pyindex 173af06f..0d323672 100644--- a/src/core_processor/board_detection/shared_board_view_detection.py+++ b/src/core_processor/board_detection/shared_board_view_detection.py@@ -1,4 +1,3 @@-def determine_shared_charuco_board_views(self):pass# # determine paired board views@@ -20,4 +19,4 @@ def determine_shared_charuco_board_views(self):# if full_charuco_detected_on_this_frame[camera_one_id] and# full_charuco_detected_on_this_frame[# camera_two_id]:- # camera_pair_joint_view_count[this_pair_num] += 1\ No newline at end of file+ # camera_pair_joint_view_count[this_pair_num] += 1diff --git a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.pyindex 03ddc484..31b13f7c 100644--- a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py+++ b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py@@ -1,10 +1,17 @@import logging+import osimport traceback+from pathlib import Path+from uuid import uuid4import cv2import mediapipe as mpfrom src.cameras.cv_camera_manager import CVCameraManager+from src.cameras.dtos.create_writer_options import (+ CreateWriterOptions,+ get_canonical_time_str,+)from src.core_processor.utils.image_fps_writer import write_fps_to_imagemp_holistic = mp.solutions.holistic@@ -19,7 +26,14 @@ class MediapipeSkeletonDetection:self._cam_manager = cam_managerasync def process_as_frame_loop(self, webcam_id, cb, model_complexity: int = 1):- with self._cam_manager.start_capture_session(webcam_id=webcam_id) as cv_cam:+ timestr = get_canonical_time_str()+ options = CreateWriterOptions(+ filename_and_ext=f"{uuid4()}.{timestr}.avi",+ relative_folder_path=Path().joinpath("skeleton_detection"),+ )+ with self._cam_manager.start_capture_session_single_cam(+ webcam_id=webcam_id, create_writer_options=options+ ) as session_obj:with mp_holistic.Holistic(min_detection_confidence=0.5,min_tracking_confidence=0.5,@@ -27,7 +41,10 @@ class MediapipeSkeletonDetection:) as holistic:try:while True:- image = self._process_single_cam_frame(holistic, cv_cam)+ image = self._process_single_cam_frame(+ holistic, session_obj.cv_cam+ )+ session_obj.writer.write(image)if cb and image is not None:await cb(image)except Exception as e:@@ -36,7 +53,12 @@ class MediapipeSkeletonDetection:raise edef process(self, model_complexity: int = 1):- with self._cam_manager.start_capture_session() as cv_cams:+ timestr = get_canonical_time_str()+ options = CreateWriterOptions(+ filename_and_ext=f"{uuid4()}.{timestr}.avi",+ parent_folder="skeleton_detection",+ )+ with self._cam_manager.start_capture_session_all_cams(options) as session_obj:with mp_holistic.Holistic(min_detection_confidence=0.5,min_tracking_confidence=0.5,@@ -48,14 +70,24 @@ class MediapipeSkeletonDetection:if exit_key == 27:logger.info("ESC has been pressed.")break- for cv_cam in cv_cams:+ for response in session_obj:+ cv_cam = response.cv_cam+ writer = response.writerimage = self._process_single_cam_frame(holistic, cv_cam)- cv2.imshow(cv_cam.webcam_id_as_str, image)+ writer.write(image)+ try:+ cv2.imshow(cv_cam.webcam_id_as_str, image)+ except:+ logger.debug(+ "Failed to draw image for some reason for cv2.imshow"+ )+ traceback.print_exc()except:logger.error("Printing traceback")traceback.print_exc()finally:- for cv_cam in cv_cams:+ for response in session_obj:+ cv_cam = response.cv_camlogger.info(f"Destroy window {cv_cam.webcam_id_as_str}")cv2.destroyWindow(cv_cam.webcam_id_as_str)cv2.waitKey(1)