Free Motion Capture for Everyone πβ¨
TypeScript
OpenCVCamera now responsible for a given camera's capture/writing needs
Jone57f08f0df36
diff --git a/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py b/src/cameras/capture/opencv_camera/camera_stream_thread_handler.pyindex fe4ea72c..4c09e47a 100644--- a/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py+++ b/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py@@ -2,33 +2,18 @@ import loggingimport threadingimport timeimport traceback-from pathlib import Path-from uuid import uuid4from src.cameras.capture.dataclasses.frame_payload import FramePayload-from src.cameras.persistence.video_writer.save_options import SaveOptions-from src.cameras.persistence.video_writer.video_writer import VideoWriter-from src.config.data_paths import freemocap_data_pathlogger = logging.getLogger(__name__)-class CameraStreamThreadHandler(threading.Thread):+class VideoCaptureThreadHandler(threading.Thread):def __init__(- self,- session_id: str,- camera_name: str,- get_next_frame,- frame_width: int,- frame_height: int,- save_video=False,+ self,+ get_next_frame,):super().__init__()- self._session_id = session_id- self._camera_name = camera_name- self._frame_width = frame_width- self._frame_height = frame_height- self._save_video = save_videoself._is_capturing_frames = Falseself._get_next_frame = get_next_frameself._num_frames_processed = 0@@ -37,19 +22,13 @@ class CameraStreamThreadHandler(threading.Thread):self.setDaemon(True)@property- def current_fps(self):+ def average_fps(self):if self._elapsed <= 0:return 0if self._num_frames_processed <= 0:return 0return self._num_frames_processed / self._elapsed- @property- def session_writer_path(self):- return Path().joinpath(- freemocap_data_path, f"{self._session_id}"- )-@propertydef latest_frame(self):return self._frame@@ -62,18 +41,14 @@ class CameraStreamThreadHandler(threading.Thread):self._is_capturing_frames = Falsedef run(self):- self._start_frame_loop(self._save_video)+ self._start_frame_loop()def _start_frame_loop(self, save_video=False):- video_writer = VideoWriter()self._is_capturing_frames = Truestart = time.time()try:while self._is_capturing_frames:frame = self._get_next_frame()- self._frame = frame- if save_video:- video_writer.write(self._frame)self._num_frames_processed += 1self._elapsed = time.time() - startexcept:@@ -81,12 +56,3 @@ class CameraStreamThreadHandler(threading.Thread):traceback.print_exc()else:logger.info("Frame loop thread exited.")- finally:- options = SaveOptions(- path_to_save_video=self.session_writer_path / 'synchronized_videos',- camera_name=self._camera_name,- frames_per_second=self.current_fps,- frame_width=self._frame_width,- frame_height=self._frame_height,- )- video_writer.save(options)diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.pyindex 8302d7b5..0c6d047b 100644--- a/src/cameras/capture/opencv_camera/opencv_camera.py+++ b/src/cameras/capture/opencv_camera/opencv_camera.py@@ -6,8 +6,9 @@ import tracebackimport cv2from src.cameras.capture.dataclasses.frame_payload import FramePayload+from src.cameras.persistence.video_writer.video_recorder import VideoRecorderfrom src.config.webcam_config import WebcamConfig-from src.cameras.capture.opencv_camera.camera_stream_thread_handler import CameraStreamThreadHandler+from src.cameras.capture.opencv_camera.camera_stream_thread_handler import VideoCaptureThreadHandlerlogger = logging.getLogger(__name__)@@ -16,20 +17,24 @@ class OpenCVCamera:"""Performant implementation of video capture against webcams"""- def __init__(self, config: WebcamConfig, session_id:str = None):- self._session_id = session_id+ def __init__(self, config: WebcamConfig, session_id: str = None):self._config = configself._name = f"Camera_ {self._config.webcam_id}"self._opencv_video_capture_object: cv2.VideoCapture = None- self._running_thread: CameraStreamThreadHandler = None- self._new_frame = False+ self._video_recorder: VideoRecorder = None+ self._running_thread: VideoCaptureThreadHandler = None+ self._is_there_a_new_frame = False@property- def new_frame(self):+ def is_there_a_new_frame(self):"""can be called to determine if the frame returned from `self.latest_frame` is new or if it has been called before"""- return self._new_frame+ return self._is_there_a_new_frame++ @property+ def video_recorder(self):+ return self._video_recorder@propertydef name(self):@@ -41,29 +46,24 @@ class OpenCVCamera:@propertydef current_fps(self):- return self._running_thread.current_fps+ return self._running_thread.average_fps@propertydef is_capturing_frames(self):if not self._running_thread:logger.info("Frame Capture thread not running yet")return False-return self._running_thread.is_capturing_frames@propertydef current_fps_short(self) -> str:- return "{:.2f}".format(self._running_thread.current_fps)+ return "{:.2f}".format(self._running_thread.average_fps)@propertydef latest_frame(self):- self._new_frame = False+ self._is_there_a_new_frame = Falsereturn self._running_thread.latest_frame- @property- def session_writer_base_path(self):- return self._running_thread.session_writer_path-def connect(self):if platform.system() == "Windows":cap_backend = cv2.CAP_DSHOW@@ -83,13 +83,24 @@ class OpenCVCamera:))return success++ try:+ image_height = image.shape[0]+ image_width = image.shape[1]+ self._video_recorder = VideoRecorder(self._name,+ image_width=image_width,+ image_height=image_height)+ except:+ logger.error(f"could not create video recorder for image with shape (image_width={image_width}, image_height={image_height}) ")+ return False+logger.debug(f"Camera found at port number {self._config.webcam_id}")fps_input_stream = int(self._opencv_video_capture_object.get(5))logger.debug("FPS of webcam hardware/input stream: {}".format(fps_input_stream))return success- def start_frame_capture(self):+ def start_frame_capture_thread(self):if self.is_capturing_frames:logger.debug(f"Already capturing frames for webcam_id: {self.webcam_id_as_str}"@@ -102,13 +113,8 @@ class OpenCVCamera:self._running_thread.start()def _create_thread(self):- return CameraStreamThreadHandler(- session_id=self._session_id,- camera_name=self._name,+ return VideoCaptureThreadHandler(get_next_frame=self.get_next_frame,- save_video=self._config.save_video,- frame_width=self.image_width,- frame_height=self.image_height,)@property@@ -154,7 +160,7 @@ class OpenCVCamera:timestamp_ns = (timestamp_ns_pre_grab + timestamp_ns_post_grab) / 2success, image = self._opencv_video_capture_object.retrieve()- self._new_frame = success+ self._is_there_a_new_frame = successreturn FramePayload(success, image, timestamp_ns)def stop_frame_capture(self):diff --git a/src/cameras/multicam_manager/cv_camera_manager.py b/src/cameras/multicam_manager/cv_camera_manager.pyindex 57750ef4..b994dbdc 100644--- a/src/cameras/multicam_manager/cv_camera_manager.py+++ b/src/cameras/multicam_manager/cv_camera_manager.py@@ -9,7 +9,7 @@ from pydantic import BaseModelfrom src.api.services.user_config import UserConfigServicefrom src.cameras.detection.cam_singleton import get_or_create_camsfrom src.cameras.capture.opencv_camera.opencv_camera import OpenCVCamera-from src.cameras.persistence.video_writer.video_writer import VideoWriter+from src.cameras.persistence.video_writer.video_recorder import VideoRecorderlogger = logging.getLogger(__name__)@@ -17,8 +17,8 @@ _open_cv_cameras = Noneclass CamAndWriterResponse(BaseModel):- cv_cam: OpenCVCamera- writer: VideoWriter+ cv_camera: OpenCVCamera+ video_recorder: VideoRecorderclass Config:arbitrary_types_allowed = True@@ -59,7 +59,7 @@ class OpenCVCameraManager:return self._open_cv_camera_objectsdef new_synchronized_frame_available(self):- new_frame_bool_list = [cam.new_frame for cam in self._open_cv_camera_objects]+ new_frame_bool_list = [cam.is_there_a_new_frame for cam in self._open_cv_camera_objects]if all(new_frame_bool_list):return Truereturn False@@ -67,7 +67,7 @@ class OpenCVCameraManager:def latest_synchronized_frame(self):synchronized_frame_dict = {}for this_cam in self._open_cv_camera_objects:- if not this_cam.new_frame:+ if not this_cam.is_there_a_new_frame:logger.error(f'New frame not available for camera {this_cam.webcam_id_as_str}')synchronized_frame_dict[this_cam.webcam_id_as_str] = Nonesynchronized_frame_dict[this_cam.webcam_id_as_str] = this_cam.latest_frame@@ -84,7 +84,7 @@ class OpenCVCameraManager:open_cv_cameras: List[OpenCVCamera] = []for this_raw_cam in raw_camera_objects:single_camera_config = self._config_service.webcam_config_by_id(this_raw_cam.webcam_id)- this_opencv_camera = OpenCVCamera(config=single_camera_config, session_id=self._session_id)+ this_opencv_camera = OpenCVCamera(config=single_camera_config)open_cv_cameras.append(this_opencv_camera)return open_cv_cameras@@ -97,9 +97,9 @@ class OpenCVCameraManager:Can capture frames from a single webcam or all webcams detected."""try:- writer = self._start_frame_capture_on_cam_id(webcam_id)- cv_cam = self.cv_cam_by_id(webcam_id)- yield CamAndWriterResponse(cv_cam=cv_cam, writer=writer)+ video_recorder = self._start_frame_capture_on_cam_id(webcam_id)+ cv_camera = self.cv_cam_by_id(webcam_id)+ yield CamAndWriterResponse(cv_camera=cv_camera, video_recorder=video_recorder)self._stop_frame_capture_all_cams()except:logger.error("Printing traceback from starting capture session by cam")@@ -108,33 +108,22 @@ class OpenCVCameraManager:@contextmanagerdef start_capture_session_all_cams(self,- ) -> ContextManager[Dict[str, CamAndWriterResponse]]:+ ) -> ContextManager[Dict[str, OpenCVCamera]]:try:- available_cam_and_writer_dict = {}- writer_dict = self._start_frame_capture_all_cams()- for webcam_id, writer in writer_dict.items():- available_cam_and_writer_dict[webcam_id] = CamAndWriterResponse(cv_cam=self.cv_cam_by_id(webcam_id),- writer=writer)- yield available_cam_and_writer_dict+ connected_cameras_dict = {}+ for cv_cam in self._open_cv_camera_objects:+ connected_cameras_dict[cv_cam.webcam_id_as_str] = cv_cam+ cv_cam.connect()+ cv_cam.start_frame_capture_thread()+ yield connected_cameras_dict+self._stop_frame_capture_all_cams()except:logger.error("Printing traceback from starting capture session by cam")traceback.print_exc()- def _start_frame_capture_all_cams(self) -> Dict[str, VideoWriter]:- d = {}- for cv_cam in self._open_cv_camera_objects:- cv_cam.connect()- d[cv_cam.webcam_id_as_str] = VideoWriter()-- # wait until all cameras are connected before starting frame capture- for cv_cam in self._open_cv_camera_objects:- cv_cam.start_frame_capture()-- return d-- def _start_frame_capture_on_cam_id(self, webcam_id: str) -> VideoWriter:+ def _start_frame_capture_on_cam_id(self, webcam_id: str):filtered_cams = list(filter(lambda c: c.webcam_id_as_str == str(webcam_id), self._open_cv_camera_objects))@@ -143,8 +132,7 @@ class OpenCVCameraManager:), "The CV Cams list should only have 1 cam per webcam_id"cv_cam = filtered_cams[0]cv_cam.connect()- cv_cam.start_frame_capture()- return VideoWriter()+ cv_cam.start_frame_capture_thread()def _stop_frame_capture_all_cams(self):for cv_cam in self._open_cv_camera_objects:diff --git a/src/cameras/persistence/video_writer/save_options.py b/src/cameras/persistence/video_writer/save_options_dataclass.pysimilarity index 100%rename from src/cameras/persistence/video_writer/save_options.pyrename to src/cameras/persistence/video_writer/save_options_dataclass.pydiff --git a/src/cameras/persistence/video_writer/video_recorder.py b/src/cameras/persistence/video_writer/video_recorder.pynew file mode 100644index 00000000..fcd8a0bb--- /dev/null+++ b/src/cameras/persistence/video_writer/video_recorder.py@@ -0,0 +1,83 @@+import logging+import os+import traceback+from pathlib import Path+from typing import List++import cv2+import numpy as np+import pandas as pd++from src.cameras.capture.dataclasses.frame_payload import FramePayload+from src.cameras.persistence.video_writer.save_options_dataclass import SaveOptions+from src.config.data_paths import freemocap_data_path++logger = logging.getLogger(__name__)+++class VideoRecorder:+ def __init__(self,+ camera_name: str,+ image_width: int,+ image_height: int,+ fourcc: str = "MP4V"+ ):+ self._camera_name = camera_name+ self._image_width = image_width+ self._image_height = image_height+ self._fourcc = fourcc+ self._frames: List[FramePayload] = []+ self._path_to_save_video = None+ self._video_type_str = ''+ self._timestamps_npy = np.array(0)++ @property+ def frame_count(self):+ return len(self._frames)++ def record(self, frame_payload: FramePayload):+ self._frames.append(frame_payload)++ def save_to_disk(self, path_to_save_video: Path, video_type_str: str = 'raw'):+ self._video_type_str = video_type_str+ self._path_to_save_video = path_to_save_video+ self._path_to_save_video.mkdir(parents=True, exist_ok=True)+ self._gather_timestamps()+ self._write_video()+ self._save_timestamps()++ def _write_video(self):+ video_file_name = self._camera_name + "_synchronized_" + self._video_type_str + ".mp4"+ video_full_path = self._path_to_save_video / video_file_name++ cv2_writer = cv2.VideoWriter(+ video_full_path,+ cv2.VideoWriter_fourcc(self._fourcc),+ self._median_framerate,+ (self._image_width, self._image_height))++ try:+ for frame in self._frames:+ cv2_writer.write(frame.image)++ except Exception as e:+ logger.debug("Failed during save in video writer")+ traceback.print_exc()+ raise e+ finally:+ logger.info(f"Saved video to path: {video_full_path}")+ cv2_writer.release()++ def _gather_timestamps(self):+ for frame in self._frames:+ self._timestamps_npy = np.append(self._timestamps_npy, frame.timestamp)+ self._median_framerate = np.median(np.diff(self._timestamps_npy))++ def _save_timestamps(self):+ timestamp_file_name_npy = self._camera_name + "_timestamps_ns.npy"+ np.save(str(self._path_to_save_video / timestamp_file_name_npy), self._timestamps_npy)+ logger.info(f"Saved timestamps to path: {timestamp_file_name_npy}")++ timestamp_file_name_csv = self._camera_name + "_timestamps_ns.csv"+ pd.DataFrame(self._timestamps_npy).to_csv(str(self._path_to_save_video / timestamp_file_name_csv))+ logger.info(f"Saved timestamps to path: {timestamp_file_name_csv}")\ No newline at end of filediff --git a/src/cameras/persistence/video_writer/video_writer.py b/src/cameras/persistence/video_writer/video_writer.pydeleted file mode 100644index 0f6cae09..00000000--- a/src/cameras/persistence/video_writer/video_writer.py+++ /dev/null@@ -1,61 +0,0 @@-import logging-import os-import traceback-from pathlib import Path-from typing import List--import cv2--from src.cameras.capture.dataclasses.frame_payload import FramePayload-from src.cameras.persistence.video_writer.save_options import SaveOptions--logger = logging.getLogger(__name__)---class VideoWriter:- def __init__(self):- self._frames: List[FramePayload] = []-- @property- def frame_count(self):- return len(self._frames)-- def write(self, frame_payload: FramePayload):- self._frames.append(frame_payload)-- def save(self, options: SaveOptions):- if not os.path.exists(options.path_to_save_video):- os.makedirs(options.path_to_save_video.resolve())- self._save_video(options)- self._save_timestamps(options)-- def _save_video(self, options: SaveOptions):- cv2_writer = self._create_cv2_video_writer(options)- try:- for frame in self._frames:- cv2_writer.write(frame.image)- except:- logger.debug("Failed during save in video writer")- traceback.print_exc()- finally:- logger.info(f"Saved video to path: {options.full_path.resolve()}")- cv2_writer.release()-- def _save_timestamps(self, options: SaveOptions):- p = Path().joinpath(options.path_to_save_video, "timestamps.txt")- try:- with open(p, "a") as fd:- for frame in self._frames:- fd.write(str(frame.timestamp) + "\n")- except Exception as e:- raise e- finally:- logger.info(f"Saved timestamps to path: {p.resolve()}")-- def _create_cv2_video_writer(self, options: SaveOptions):- return cv2.VideoWriter(- str(options.full_path.resolve()),- cv2.VideoWriter_fourcc(*options.fourcc),- options.frames_per_second,- (options.frame_width, options.frame_height),- )diff --git a/src/session_pipeline/session_pipeline_orchestrator.py b/src/session_pipeline/session_pipeline_orchestrator.pyindex c4ba6a5e..45e997c7 100644--- a/src/session_pipeline/session_pipeline_orchestrator.py+++ b/src/session_pipeline/session_pipeline_orchestrator.py@@ -7,7 +7,8 @@ import cv2import numpy as npfrom src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager-from src.cameras.persistence.video_writer.save_options import SaveOptions+from src.cameras.persistence.video_writer.save_options_dataclass import SaveOptions+from src.config.data_paths import freemocap_data_pathfrom src.core_processor.camera_calibration.camera_calibrator import CameraCalibratorfrom src.core_processor.fps.fps_counter import FPSCamCounterfrom src.core_processor.mediapipe_skeleton_detection.mediapipe_skeleton_detection import MediaPipeSkeletonDetection@@ -30,14 +31,18 @@ class SessionPipelineOrchestrator:self._camera_calibrator = CameraCalibrator()self._mediapipe_skeleton_detector = MediaPipeSkeletonDetection(model_complexity=0)+ @property+ def session_folder_path(self):+ return freemocap_data_path / self._session_id+async def process_by_cam_id(self, webcam_id: str, cb):with self._open_cv_camera_manager.start_capture_session_single_cam(webcam_id=webcam_id,) as session_obj:fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)fps_manager.start_all()- cv_cam = session_obj.cv_cam- writer = session_obj.writer+ cv_cam = session_obj.cv_camera+ writer = session_obj.video_recordertry:while True:if not cv_cam.is_capturing_frames:@@ -48,14 +53,14 @@ class SessionPipelineOrchestrator:charuco_frame_payload = self._camera_calibrator.calibrate(cv_cam)if cb and charuco_frame_payload.annotated_image is not None:- writer.write(charuco_frame_payload.raw_frame_payload)+ writer.record(charuco_frame_payload.raw_frame_payload)await cb(charuco_frame_payload.annotated_image)except Exception as e:logger.error("Printing traceback")traceback.print_exc()raise efinally:- writer = session_obj.writer+ writer = session_obj.video_recorderoptions = SaveOptions(writer_dir=Path().joinpath(cv_cam.session_writer_base_path,@@ -66,7 +71,7 @@ class SessionPipelineOrchestrator:frame_width=cv_cam.image_width(),frame_height=cv_cam.image_height(),)- writer.save(options)+ writer.save_to_disk(options)def run(self,@@ -81,7 +86,7 @@ class SessionPipelineOrchestrator:If return images is true, the images are returned to the caller"""- with self._open_cv_camera_manager.start_capture_session_all_cams() as available_cam_and_writer_dict:+ with self._open_cv_camera_manager.start_capture_session_all_cams() as connected_cameras_dict:try:fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)@@ -97,24 +102,21 @@ class SessionPipelineOrchestrator:this_frame_timestamps_dict = {}incoming_synchronized_frame_dict = self._open_cv_camera_manager.latest_synchronized_frame()- for this_webcam_id, this_cam_and_writer in available_cam_and_writer_dict.items():-- this_open_cv_camera = this_cam_and_writer.cv_cam- this_video_writer_object = this_cam_and_writer.writer+ for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():- this_cam_latest_frame = incoming_synchronized_frame_dict[this_webcam_id]+ this_cam_latest_sync_frame = incoming_synchronized_frame_dict[this_webcam_id]- if this_cam_latest_frame is None:- image_to_display = this_open_cv_camera.latest_frame- this_frame_timestamps_dict[this_webcam_id] = np.nan+ if this_cam_latest_sync_frame is None:+ image_to_display = this_open_cv_camera.latest_frame #if this camera has no new frame, write the previous one instead+ this_frame_timestamps_dict[this_webcam_id] = np.nan # a `nan` timestamp denotes a dropped frameelse:this_frame_timestamps_dict[- this_webcam_id] = (this_cam_latest_frame.timestamp - self._session_start_time)/1e6 #milliseconds, I think- image_to_display = this_cam_latest_frame.image+ this_webcam_id] = (this_cam_latest_sync_frame.timestamp - self._session_start_time)/1e6 #milliseconds, I think+ image_to_display = this_cam_latest_sync_frame.imageif save_video:- print('sending frame to be written')- this_video_writer_object.write(this_cam_latest_frame)+ # print('sending frame to be written')+ this_open_cv_camera.video_recorder.record(this_cam_latest_sync_frame)if calibrate_cameras:undistorted_annotated_image = self._camera_calibrator.calibrate(this_open_cv_camera)@@ -138,20 +140,9 @@ class SessionPipelineOrchestrator:logger.error("Printing traceback")traceback.print_exc()finally:- for this_webcam_id, this_cam_and_writer in available_cam_and_writer_dict.items():- this_open_cv_camera = this_cam_and_writer.cv_cam- # this_video_writer_object = this_cam_and_writer.writer- # options = SaveOptions(- # path_to_save_video=Path().joinpath(- # this_open_cv_camera.session_writer_base_path,- # "charuco_board_detection",- # f"webcam_{this_open_cv_camera.webcam_id_as_str}",- # ),- # fps=fps_manager.current_fps_for(this_open_cv_camera.webcam_id_as_str),- # frame_width=this_open_cv_camera.image_width,- # frame_height=this_open_cv_camera.image_height,- # )- # this_video_writer_object.save(options)+ for this_open_cv_camera in connected_cameras_dict.values():+ if save_video:+ this_open_cv_camera.video_recorder.save_to_disk(self.session_folder_path / 'synchronized_videos')logger.info(f"Destroy window {this_open_cv_camera.webcam_id_as_str}")cv2.destroyWindow(this_open_cv_camera.webcam_id_as_str)cv2.waitKey(1)diff --git a/src/tests/video_writer/test_video_writer.py b/src/tests/video_writer/test_video_writer.pyindex 5b83329c..a5ee370d 100644--- a/src/tests/video_writer/test_video_writer.py+++ b/src/tests/video_writer/test_video_writer.py@@ -7,7 +7,7 @@ from unittest import TestCaseimport numpy as npfrom src.cameras.capture.dataclasses.frame_payload import FramePayload-from src.cameras.persistence.video_writer.video_writer import SaveOptions, VideoWriter+from src.cameras.persistence.video_writer.video_recorder import SaveOptions, VideoRecorderclass VideoWriterTestCase(TestCase):@@ -22,8 +22,8 @@ class VideoWriterTestCase(TestCase):example_payload = FramePayload(image=np.random.randint(0, 4, (720, 1280, 3)), timestamp=time.time_ns())- vw = VideoWriter()- vw.write(example_payload)+ vw = VideoRecorder()+ vw.record(example_payload)assert vw.frame_count == 1@@ -32,8 +32,8 @@ class VideoWriterTestCase(TestCase):example_payload = FramePayload(image=np.random.randint(0, 4, (720, 1280, 3)), timestamp=time.time_ns())- vw = VideoWriter()- vw.write(example_payload)+ vw = VideoRecorder()+ vw.record(example_payload)save_options = SaveOptions(writer_dir=file_path,@@ -43,7 +43,7 @@ class VideoWriterTestCase(TestCase):fps=1,)- vw.save(save_options)+ vw.save_to_disk(save_options)expected_path = Path().joinpath(file_path, "movie.mp4")assert save_options.full_path == expected_path, "Paths do not match"