Free Motion Capture for Everyone πβ¨
TypeScript
some kinda progress
Jon2baaff176526
diff --git a/requirements.txt b/requirements.txtindex e5286eca..d7ebd8e2 100644--- a/requirements.txt+++ b/requirements.txt@@ -34,4 +34,5 @@ pyqt5==5.15.6pyopengl==3.1.6mediapipe~=0.8.9.1setuptools~=58.0.4-aniposelib~=0.4.3\ No newline at end of file+aniposelib~=0.4.3+rich~=12.3.0\ No newline at end of filediff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.pyindex 9577557b..90ed3ba7 100644--- a/src/cameras/capture/opencv_camera/opencv_camera.py+++ b/src/cameras/capture/opencv_camera/opencv_camera.py@@ -115,16 +115,13 @@ class OpenCVCamera:else:image_height = image.shape[0]image_width = image.shape[1]- try:- self._video_recorder = VideoRecorder(self._name,- image_width=image_width,- image_height=image_height,- session_id=self.session_id,- calibration_video_bool=self._calibration_video_bool)- except:- logger.error(- f"could not create video recorder for image with shape (image_width={image_width}, image_height={image_height}) ")- return False # NOTE- this is a failure because we *tried* to make a video_recorder and it didn't work++ self._video_recorder = VideoRecorder(self._name,+ image_width=image_width,+ image_height=image_height,+ session_id=self.session_id,+ calibration_video_bool=self._calibration_video_bool)+return successdiff --git a/src/cameras/multicam_manager/cv_camera_manager.py b/src/cameras/multicam_manager/cv_camera_manager.pyindex 426f1bef..d0e7732d 100644--- a/src/cameras/multicam_manager/cv_camera_manager.py+++ b/src/cameras/multicam_manager/cv_camera_manager.py@@ -1,8 +1,10 @@import logging+import timeimport tracebackfrom contextlib import contextmanager-from typing import ContextManager, Dict, List, Optional+from typing import ContextManager, Dict, List, Optional, Union+import numpy as npfrom pydantic import BaseModelfrom src.api.services.user_config import UserConfigService@@ -10,6 +12,8 @@ from src.cameras.capture.opencv_camera.opencv_camera import OpenCVCamerafrom src.cameras.detection.cam_singleton import get_or_create_camsfrom src.cameras.persistence.video_writer.video_recorder import VideoRecorderfrom src.config.webcam_config import WebcamConfig+from src.core_processor.fps.timestamp_manager import TimestampManager+from src.pipelines.session_pipeline.data_classes.multi_frame_payload import MultiFramePayloadlogger = logging.getLogger(__name__)@@ -23,44 +27,79 @@ class CamAndWriterResponse(BaseModel):class OpenCVCameraManager:- def __init__(self, session_id: str = None):+ def __init__(self,+ session_id: str = None,+ expected_framerate: Union[int, float] = None,+ ):self._session_id = session_idself._config_service = UserConfigService()self._detected_cams_data = get_or_create_cams()+ self._session_start_time_unix_ns = None+ self._timestamp_manager: Union[TimestampManager, None] = None+ self._connected_cameras_dict = {}+ self._expected_framerate = expected_framerate++ @property+ def timestamp_manager(self):+ if self._timestamp_manager is None:+ logger.error('timestamp manager has not been created yet')+ else:+ return self._timestamp_manager@propertydef available_webcam_ids(self):return [cam_id.webcam_id for cam_id in get_or_create_cams().cameras_found_list]- def _create_opencv_cameras(self, calibration_videos:bool=False):- raw_camera_objects = self._detected_cams_data.cameras_found_list- open_cv_cameras: List[OpenCVCamera] = []- for this_raw_cam in raw_camera_objects:- opencv_cam_obj = self._create_single_opencv_cam(this_raw_cam.webcam_id, calibration_video_bool =calibration_videos)- open_cv_cameras.append(opencv_cam_obj)- return open_cv_cameras-- def _create_single_opencv_cam(self, webcam_id: str, calibration_video_bool:bool =False):- webcam_config_model = self._config_service.webcam_config_by_id(webcam_id, self._session_id)- single_camera_config = WebcamConfig(webcam_id=webcam_config_model.webcam_id,- exposure=webcam_config_model.exposure,- resolution_width=webcam_config_model.resolution_width,- resolution_height=webcam_config_model.resolution_height, )- return OpenCVCamera(config=single_camera_config,- session_id=self._session_id,- calibration_video_bool = calibration_video_bool)+ @property+ def latest_multi_frame(self) -> Union[MultiFramePayload,None]:+ if not self.new_multi_frame_ready():+ logging.error("a multi_frame was requested before it was ready!")+ raise Exception++ this_multi_frame_dict = {}+ for this_cam in self._connected_cameras_dict.values():+ if not this_cam.new_frame_ready:+ logger.error('It shouldnt be able to get here if the `new_multi_frame`')+ this_multi_frame_dict[this_cam.webcam_id_as_str] = this_cam.latest_frame+ self._timestamp_manager.log_new_multi_frame_timestamp_ns(time.perf_counter_ns())++ if not self._timestamp_manager.verify_multi_frame_is_synchronized(this_multi_frame_dict,+ self._expected_framerate):+ m_f_interval = self._timestamp_manager.latest_multi_frame_interval+ m_f_number = self._timestamp_manager.multi_frame_timestamp_logger.number_of_frames+ logger.error(f"Multi frame is not synchronized!! Skipping this one - multi_frame_frame_number|interval: {m_f_number}|{m_f_interval:.3f}")+ # raise Exception+ return None++ return MultiFramePayload(frames_dict=this_multi_frame_dict,+ multi_frame_number=self._timestamp_manager.multi_frame_timestamp_logger.number_of_frames,+ intra_frame_interval=self._timestamp_manager.latest_multi_frame_interval,+ each_frame_timestamp=self._timestamp_manager.latest_multi_frame_timestamp_list,+ multi_frame_timestamp=self._timestamp_manager.multi_frame_timestamp_logger.latest_timestamp,+ )++ def new_multi_frame_ready(self):+ """cycle through connected cameras and return false if one isn't read yet+ TODO - make another kind of check that uses a clock and sends an empty frame if one camera hasn't yeild a frame within a reasonable time period. With this method, the whol camera system will stall if one camera stops producing frames"""+ for this_cam in self._connected_cameras_dict.values():+ if not this_cam.new_frame_ready:+ return False+ return True@contextmanagerdef start_capture_session_single_cam(- self, webcam_id: str+ self, webcam_id: str) -> ContextManager[CamAndWriterResponse]:"""Context manager for easy start up, usage, and cleanup of camera resources.Can capture frames from a single webcam or all webcams detected."""+cv_camera = self._create_single_opencv_cam(webcam_id)+ self._connected_cameras_dict['0'] = cv_cameratry:self._start_frame_capture_on_cam_id(cv_camera)+ self._initialize_timestamp_logger() # start timestamp logger and whayield CamAndWriterResponse(cv_camera=cv_camera)self._stop_frame_capture([cv_camera])except:@@ -69,23 +108,49 @@ class OpenCVCameraManager:@contextmanagerdef start_capture_session_all_cams(- self,+ self,calibration_videos=False,) -> ContextManager[Dict[str, OpenCVCamera]]:-- open_cv_camera_objects = self._create_opencv_cameras(calibration_videos=True)+ open_cv_camera_objects = self._create_opencv_cameras(calibration_videos=calibration_videos)try:- connected_cameras_dict = {}+for cv_cam in open_cv_camera_objects:- connected_cameras_dict[cv_cam.webcam_id_as_str] = cv_cam+ self._connected_cameras_dict[cv_cam.webcam_id_as_str] = cv_camself._start_frame_capture_on_cam_id(cv_cam)- yield connected_cameras_dict+ self._initialize_timestamp_logger()+ yield self._connected_cameras_dictself._stop_frame_capture(open_cv_camera_objects)except:logger.error("Printing traceback from starting capture session by cam")traceback.print_exc()+ def _create_opencv_cameras(self, calibration_videos: bool = False):+ raw_camera_objects = self._detected_cams_data.cameras_found_list+ open_cv_cameras: List[OpenCVCamera] = []+ for this_raw_cam in raw_camera_objects:+ opencv_cam_obj = self._create_single_opencv_cam(this_raw_cam.webcam_id,+ calibration_video_bool=calibration_videos)+ open_cv_cameras.append(opencv_cam_obj)+ return open_cv_cameras++ def _create_single_opencv_cam(self, webcam_id: str, calibration_video_bool: bool = False):+ webcam_config_model = self._config_service.webcam_config_by_id(webcam_id, self._session_id)+ single_camera_config = WebcamConfig(webcam_id=webcam_config_model.webcam_id,+ exposure=webcam_config_model.exposure,+ resolution_width=webcam_config_model.resolution_width,+ resolution_height=webcam_config_model.resolution_height, )+ return OpenCVCamera(config=single_camera_config,+ session_id=self._session_id,+ calibration_video_bool=calibration_video_bool)++ def _initialize_timestamp_logger(self):+ self._session_start_time_unix_ns = time.time_ns()+ self._session_start_time_perf_counter_ns = time.perf_counter_ns()+ self._timestamp_manager = TimestampManager(self.available_webcam_ids,+ self._session_start_time_unix_ns,+ self._session_start_time_perf_counter_ns)+def _start_frame_capture_on_cam_id(self, opencv_cam: OpenCVCamera):opencv_cam.connect()opencv_cam.start_frame_capture_thread()diff --git a/src/cameras/persistence/video_writer/video_recorder.py b/src/cameras/persistence/video_writer/video_recorder.pyindex 34ccb177..f887aff0 100644--- a/src/cameras/persistence/video_writer/video_recorder.py+++ b/src/cameras/persistence/video_writer/video_recorder.py@@ -2,7 +2,7 @@ import loggingimport osimport tracebackfrom pathlib import Path-from typing import List+from typing import List, Unionimport cv2import numpy as np@@ -11,44 +11,53 @@ import pandas as pdfrom src.cameras.capture.dataclasses.frame_payload import FramePayloadfrom src.cameras.persistence.video_writer.save_options_dataclass import SaveOptionsfrom src.config.data_paths import freemocap_data_path-from src.config.home_dir import get_session_path+from src.config.home_dir import get_session_path, get_synchronized_videos_path, get_calibration_videos_path, \+ get_mediapipe_annotated_videos_path+from rich.progress import Progresslogger = logging.getLogger(__name__)class VideoRecorder:def __init__(self,- camera_name: str,+ video_name: str,image_width: int,image_height: int,session_id: str,fourcc: str = "MP4V",- calibration_video_bool:bool=False,+ calibration_video_bool: bool = False,+ mediapipe_annotated_video_bool: bool = False,):- self._camera_name = camera_name+ self._video_name = video_nameself._image_width = image_widthself._image_height = image_heightself._fourcc = fourcc- self._frame_list: List[FramePayload] = []- self._video_type_str = ''+ self._frame_payload_list: List[FramePayload] = []self._timestamps_npy = np.empty(0)self._cv2_video_writer = None# get yr paths straight- session_path = Path(get_session_path(session_id))- if calibration_video_bool:- self._synchronized_videos_folder_path = session_path / 'calibration_videos'+ self._session_id = session_id+ session_path = Path(get_session_path(self._session_id))++ if mediapipe_annotated_video_bool:+ self._video_folder_path = Path(get_mediapipe_annotated_videos_path(self._session_id))+ elif calibration_video_bool:+ self._video_folder_path = Path(get_calibration_videos_path(self._session_id))else:- self._synchronized_videos_folder_path = session_path / 'synchronized_videos'+ self._video_folder_path = Path(get_synchronized_videos_path(self._session_id))- self._synchronized_videos_folder_path.mkdir(parents=True, exist_ok=True)- video_file_name = self._camera_name + '.mp4'- self._path_to_save_video_file = self._synchronized_videos_folder_path / video_file_name+ self._video_folder_path.mkdir(parents=True, exist_ok=True)+ video_file_name = self._video_name + '.mp4'+ self._path_to_save_video_file = self._video_folder_path / video_file_name+ @property+ def video_name(self):+ return self._video_name@property- def synchronized_videos_folder_path(self):- return self._synchronized_videos_folder_path+ def video_folder_path(self):+ return self._video_folder_path@propertydef path_to_save_video_file(self):@@ -56,20 +65,20 @@ class VideoRecorder:@propertydef frame_count(self):- return len(self._frame_list)+ return len(self._frame_payload_list)@propertydef median_framerate(self):- if self.frame_count < 10:- logger.warning(f"camera framerate not known, setting video playback rate to 30 frames per second")- self._median_framerate = 30+ if self.frame_count == 0:+ logger.error(f"No Frames to save for {self.path_to_save_video_file}")+ raise Exceptionelse:self._gather_timestamps()self._median_framerate = (np.nanmedian(np.diff(self._timestamps_npy / 1e9))) ** -1return self._median_framerate- def save_frame_to_video_file(self,frame_payload: FramePayload):+ def save_frame_payload_to_video_file(self, frame_payload: FramePayload):if self._cv2_video_writer is None:self._initialize_video_writer()@@ -78,12 +87,12 @@ class VideoRecorder:def close(self):self._cv2_video_writer.release()- def append_frame_to_list(self, frame_payload: FramePayload):- self._frame_list.append(frame_payload)+ def append_frame_payload_to_list(self, frame_payload: FramePayload):+ self._frame_payload_list.append(frame_payload)- def save_frame_list_to_disk(self):- if len(self._frame_list) == 0:- logging.error(f"No frames to save for camera: {self._camera_name}")+ def save_frame_payload_list_to_disk(self):+ if len(self._frame_payload_list) == 0:+ logging.error(f"No frames to save for camera: {self._video_name}")returnself._gather_timestamps()@@ -91,18 +100,32 @@ class VideoRecorder:self._write_frame_list_to_video_file()self._save_timestamps()- def _initialize_video_writer(self):+ def save_image_list_to_disk(self,+ image_list: List[np.ndarray],+ frame_rate: Union[int, float] = None):+ if len(image_list) == 0:+ logging.error(f"No frames to save for : {self._video_name}")+ return++ self._initialize_video_writer(frames_per_second=frame_rate)+ self._write_image_list_to_video_file(image_list)++ def _initialize_video_writer(self, frames_per_second: Union[int, float] = None):++ if frames_per_second is None:+ frames_per_second = self.median_framerate+ else:+ frames_per_second = frames_per_secondself._cv2_video_writer = cv2.VideoWriter(str(self.path_to_save_video_file),cv2.VideoWriter_fourcc(*self._fourcc),- self.median_framerate,- (self._image_width, self._image_height))+ frames_per_second,+ (int(self._image_width), int(self._image_height)))def _write_frame_list_to_video_file(self):-try:- for frame in self._frame_list:+ for frame in self._frame_payload_list:self._cv2_video_writer.write(frame.image)except Exception as e:@@ -113,21 +136,35 @@ class VideoRecorder:logger.info(f"Saved video to path: {self.path_to_save_video_file}")self._cv2_video_writer.release()+ def _write_image_list_to_video_file(self, image_list:List[np.ndarray]):+ try:+ for image in image_list:+ self._cv2_video_writer.write(image)+++ except Exception as e:+ logger.error(f"Failed during save in video writer: {self.path_to_save_video_file}")+ traceback.print_exc()+ raise e+ finally:+ logger.info(f"Saved video to path: {self.path_to_save_video_file}")+ self._cv2_video_writer.release()+def _gather_timestamps(self):try:- for frame in self._frame_list:+ for frame in self._frame_payload_list:self._timestamps_npy = np.append(self._timestamps_npy, frame.timestamp)except:logger.error("Error gathering timestamps")def _save_timestamps(self):- timestamp_file_name_npy = self._camera_name + "_timestamps_binary.npy"- timestamp_npy_full_save_path = self._synchronized_videos_folder_path / timestamp_file_name_npy+ timestamp_file_name_npy = self._video_name + "_timestamps_binary.npy"+ timestamp_npy_full_save_path = self._video_folder_path / timestamp_file_name_npynp.save(str(timestamp_npy_full_save_path), self._timestamps_npy)logger.info(f"Saved timestamps to path: {timestamp_file_name_npy}")- timestamp_file_name_csv = self._camera_name + "_timestamps_human_readable.csv"- timestamp_csv_full_save_path = self._synchronized_videos_folder_path / timestamp_file_name_csv+ timestamp_file_name_csv = self._video_name + "_timestamps_human_readable.csv"+ timestamp_csv_full_save_path = self._video_folder_path / timestamp_file_name_csvtimestamp_dataframe = pd.DataFrame(self._timestamps_npy)timestamp_dataframe.to_csv(str(timestamp_csv_full_save_path))logger.info(f"Saved timestamps to path: {timestamp_file_name_csv}")diff --git a/src/config/home_dir.py b/src/config/home_dir.pyindex 5621cd80..fc173639 100644--- a/src/config/home_dir.py+++ b/src/config/home_dir.py@@ -22,3 +22,15 @@ def get_session_path(session_id: str):base_save_path = Path(get_freemocap_data_folder_path())session_path = base_save_path / session_idreturn str(session_path)++def get_synchronized_videos_path(session_id: str):+ synchronized_videos_path = Path(get_session_path(session_id)) / 'synchronized_videos'+ return str(synchronized_videos_path)++def get_calibration_videos_path(session_id: str):+ calibration_videos_path = Path(get_session_path(session_id)) / 'calibration_videos'+ return str(calibration_videos_path)++def get_mediapipe_annotated_videos_path(session_id: str):+ mediapipe_annotated_videos_path = Path(get_session_path(session_id)) / 'mediapipe_annotated_videos'+ return str(mediapipe_annotated_videos_path)diff --git a/src/core_processor/fps/timestamp_manager.py b/src/core_processor/fps/timestamp_manager.pyindex 04c391a0..e2d3445e 100644--- a/src/core_processor/fps/timestamp_manager.py+++ b/src/core_processor/fps/timestamp_manager.py@@ -1,16 +1,19 @@+import loggingfrom typing import Dict, List, Unionimport numpy as np+logger = logging.getLogger(__name__)+class TimestampLogger:_num_frames_processed: int_start_time: float- def __init__(self, _session_start_time_unix_ns: int):- self._session_start_time_unix_ns = _session_start_time_unix_ns+ def __init__(self, _session_start_time_perf_counter_ns: int):+ self._session_start_time_perf_counter_ns = _session_start_time_perf_counter_nsself._num_frames_processed = 0- self._timestamps = np.empty(0)+ self._timestamps_ns = np.empty(0)def is_started(self):return self._num_frames_processed > 0@@ -19,41 +22,74 @@ class TimestampLogger:def median_frames_per_second(self):if self._num_frames_processed <= 10:return 0- return np.nanmedian(np.diff(self._timestamps) / 1e9) ** -1+ return np.nanmedian(np.diff(self._timestamps_ns) / 1e9) ** -1@propertydef number_of_frames(self):return self._num_frames_processed@property- def timestamps(self):+ def timestamps_unix_ns(self):"""return timestamps in nanoseconds in Unix epoch format (which is both the most useful, ugliest, AND funniest time format around. You can use `self.timestamps_in_seconds_from_zero()` for a more intuitive time format"""- return self._timestamps+ return self._timestamps_ns - self._session_start_time_perf_counter_ns++ @property+ def timestamps(self):+ return self.timestamps_in_seconds_from_session_start@propertydef timestamps_in_seconds_from_session_start(self):"""timestamps in seconds counting up from zero (where zero is the start time of the recording session)"""- return (self._timestamps - self._session_start_time_unix_ns) / 1e9+ return (self._timestamps_ns - self._session_start_time_perf_counter_ns) / 1e9@property- def last_timestamp(self):- if self._timestamps.shape[0] > 0:- return self._timestamps[-1]+ def latest_timestamp(self):+ """returns timestamps in seconds from session start"""+ if self.timestamps.shape[0] > 0:+ return self.timestamps[-1]return None- def log_new_timestamp(self, timestamp):- self._timestamps = np.append(self._timestamps, timestamp)+ def log_new_timestamp_ns(self, timestamp):+ self._timestamps_ns = np.append(self._timestamps_ns, timestamp)self._num_frames_processed += 1class TimestampManager:- def __init__(self, webcam_ids: List[str], session_start_time_unix_ns: int):- self._session_start_time_unix_ns = session_start_time_unix_ns+ def __init__(self,+ webcam_ids: List[str],+ session_start_time_unix_ns: int,+ _session_start_time_perf_counter_ns:int):+ self._session_start_time_unix_ns = session_start_time_unix_ns #time.time_ns(), UNIX epoch time, nanoseconds+ self._session_start_time_perf_counter_ns = _session_start_time_perf_counter_ns #time.perf_counter_ns, arbitrary timebase, nanoseconds - corresponds to self.session_start_time_nsself._webcam_ids = webcam_ids- self._webcam_timestamp_loggers: Dict[str, TimestampLogger] = self._initialize_timestamp_loggers(webcam_ids)- self._main_loop_timestamp_logger = TimestampLogger(self._session_start_time_unix_ns)+ self._webcam_timestamp_loggers: Dict[str, TimestampLogger] = self._initialize_webcam_timestamp_loggers(+ webcam_ids)+ self._main_loop_timestamp_logger = TimestampLogger(self._session_start_time_perf_counter_ns)+ self._multi_frame_timestamp_logger = TimestampLogger(self._session_start_time_perf_counter_ns)+ self._multi_frame_timestamps_list = []+ self._multi_frame_interval_list = []++ @property+ def multi_frame_timestamp_logger(self):+ return self._multi_frame_timestamp_logger++ @property+ def multi_frame_timestamps_list(self):+ return self._multi_frame_timestamps_list++ @property+ def multi_frame_interval_list(self):+ return self._multi_frame_interval_list++ @property+ def latest_multi_frame_timestamp_list(self):+ return self._multi_frame_timestamps_list[-1]++ @property+ def latest_multi_frame_interval(self):+ return self._multi_frame_interval_list[-1]- def _initialize_timestamp_loggers(self, webcam_ids: List[str]):+ def _initialize_webcam_timestamp_loggers(self, webcam_ids: List[str]):dictionary_of_timestamp_logger_objects = {}for webcam_id in webcam_ids:dictionary_of_timestamp_logger_objects[webcam_id] = TimestampLogger(self._session_start_time_unix_ns)@@ -62,20 +98,55 @@ class TimestampManager:def get_timestamps_from_camera_in_seconds_from_session_start(self, webcam_id):return self._webcam_timestamp_loggers[webcam_id].timestamps_in_seconds_from_session_start- def get_number_of_frames(self, webcam_id):+ def number_of_frames(self, webcam_id):return self._webcam_timestamp_loggers[webcam_id].number_of_frames- def increment_main_loop_timestamp_logger(self, timestamp):- self._main_loop_timestamp_logger.log_new_timestamp(timestamp)-def median_frames_per_second_for_main_loop(self):return self._main_loop_timestamp_logger.median_frames_per_second- def increment_frame_processed_for_webcam(self, webcam_id: str, timestamp: Union[float, int],- frame_number: int = None):- self._webcam_timestamp_loggers[webcam_id].log_new_timestamp(timestamp)- if frame_number is not None:- assert (self._webcam_timestamp_loggers[webcam_id].number_of_frames == frame_number, f"Numer of timestamps ({self._webcam_timestamp_loggers[webcam_id].number_of_frames}) logged does not match frame_number ({frame_number}) for: camera_{webcam_id}")-def median_frames_per_second_for_webcam(self, webcam_id):return self._webcam_timestamp_loggers[webcam_id].median_frames_per_second++ def log_new_timestamp_for_main_loop_ns(self, timestamp):+ self._main_loop_timestamp_logger.log_new_timestamp_ns(timestamp)++ def log_new_multi_frame_timestamp_ns(self, timestamp):+ self._multi_frame_timestamp_logger.log_new_timestamp_ns(timestamp)++ def log_new_timestamp_for_webcam_ns(self, webcam_id: str, timestamp: Union[float, int],+ frame_number: int = None):+ self._webcam_timestamp_loggers[webcam_id].log_new_timestamp_ns(timestamp)+ if frame_number is not None:+ assert (self._webcam_timestamp_loggers[webcam_id].number_of_frames == frame_number,+ f"Numer of timestamps ({self._webcam_timestamp_loggers[webcam_id].number_of_frames}) logged does not match frame_number ({frame_number}) for: camera_{webcam_id}")++ def _get_each_frame_timestamp(self, this_multi_frame_dict) -> List:++ this_multi_frame_timestamps = []+ for this_cam_id, this_cam_frame in this_multi_frame_dict.items():+ this_multi_frame_timestamps.append(this_cam_frame.timestamp)+ return this_multi_frame_timestamps++ def _get_intra_frame_interval_in_sec(self, this_multi_frame_timestamps) -> float:+ timestamp_min = np.min(this_multi_frame_timestamps)+ timestamp_max = np.max(this_multi_frame_timestamps)+ return (timestamp_max - timestamp_min) / 1e9++ def verify_multi_frame_is_synchronized(self,+ this_multi_frame_dict,+ expected_framerate: Union[int, None]) -> bool:++ this_multi_frame_timestamps = self._get_each_frame_timestamp(this_multi_frame_dict)+ this_multiframe_timestamp_interval_sec = self._get_intra_frame_interval_in_sec(this_multi_frame_timestamps)++ self._multi_frame_timestamps_list.append(this_multi_frame_timestamps)+ self._multi_frame_interval_list.append(this_multiframe_timestamp_interval_sec)++ if expected_framerate is None:+ logger.warning('`expected_framerate` not specified, Cannot verify multi_frame synchronization')+ return True++ if this_multiframe_timestamp_interval_sec > 2*(expected_framerate ** -1):+ return False++ return Truediff --git a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.pydeleted file mode 100644index 69971e30..00000000--- a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py+++ /dev/null@@ -1,65 +0,0 @@-from dataclasses import dataclass--import numpy as np-import mediapipe as mp---@dataclass-class MediaPipeSkeletonDetection:- model_complexity: int = 1 # can be 0,1, or 2 - higher numbers are more accurate but heavier computationally- min_detection_confidence: float = .5- min_tracking_confidence: float = .5-- def __post_init__(self):- self._mp_drawing = mp.solutions.drawing_utils- self._mp_drawing_styles = mp.solutions.drawing_styles- self._mp_holistic = mp.solutions.holistic-- self._body_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=.2)- self._hand_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=.2)- self._face_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=.2)-- self._holistic_tracker = self._mp_holistic.Holistic(model_complexity=self.model_complexity,- min_detection_confidence=self.min_detection_confidence,- min_tracking_confidence=self.min_tracking_confidence)-- def detect_skeleton_in_image(self, raw_image, annotate_image=True):- mediapipe_results = self._holistic_tracker.process(raw_image) # <-this is where the magic happens- if annotate_image:- return self._annotate_image(raw_image, mediapipe_results)--- def _annotate_image(self, image, mediapipe_results):- self._mp_drawing.draw_landmarks(image=image,- landmark_list=mediapipe_results.face_landmarks,- connections=self._mp_holistic.FACEMESH_CONTOURS,- landmark_drawing_spec=None,- connection_drawing_spec=self._mp_drawing_styles- .get_default_face_mesh_contours_style())- self._mp_drawing.draw_landmarks(image=image,- landmark_list=mediapipe_results.face_landmarks,- connections=self._mp_holistic.FACEMESH_TESSELATION,- landmark_drawing_spec=None,- connection_drawing_spec=self._mp_drawing_styles- .get_default_face_mesh_tesselation_style())- self._mp_drawing.draw_landmarks(image=image,- landmark_list=mediapipe_results.pose_landmarks,- connections=self._mp_holistic.POSE_CONNECTIONS,- landmark_drawing_spec=self._mp_drawing_styles- .get_default_pose_landmarks_style())- self._mp_drawing.draw_landmarks(- image=image,- landmark_list=mediapipe_results.left_hand_landmarks,- connections=self._mp_holistic.HAND_CONNECTIONS,- landmark_drawing_spec=None,- connection_drawing_spec=self._mp_drawing_styles- .get_default_hand_connections_style())-- self._mp_drawing.draw_landmarks(- image=image,- landmark_list=mediapipe_results.right_hand_landmarks,- connections=self._mp_holistic.HAND_CONNECTIONS,- landmark_drawing_spec=None,- connection_drawing_spec=self._mp_drawing_styles- .get_default_hand_connections_style())- return imagediff --git a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.pynew file mode 100644index 00000000..2f9a7d69--- /dev/null+++ b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py@@ -0,0 +1,125 @@+import logging+from dataclasses import dataclass+from pathlib import Path+from typing import List++import cv2+import numpy as np+import mediapipe as mp+from rich.progress import Progress++from jon_scratch.opencv_camera import TweakedModel+from src.cameras.capture.dataclasses.frame_payload import FramePayload+from src.cameras.persistence.video_writer.video_recorder import VideoRecorder+from src.config.home_dir import get_session_path, get_synchronized_videos_path+logger = logging.getLogger(__name__)+++class MediaPipeSkeletonDetector:+ def __init__(self, session_id: str = None):+ self._session_id = session_id+ self.model_complexity = 2 # can be 0,1, or 2 - higher numbers are more accurate but heavier computationally+ self.min_detection_confidence = .5+ self.min_tracking_confidence = .5++ self._mediapipe_payload_list=[]++ self._mp_drawing = mp.solutions.drawing_utils+ self._mp_drawing_styles = mp.solutions.drawing_styles+ self._mp_holistic = mp.solutions.holistic++ self._body_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=1)+ self._hand_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=1)+ self._face_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=1)++ self._holistic_tracker = self._mp_holistic.Holistic(model_complexity=self.model_complexity,+ min_detection_confidence=self.min_detection_confidence,+ min_tracking_confidence=self.min_tracking_confidence)++ def detect_skeleton_in_image(self, raw_image, annotate_image=True):+ mediapipe_results = self._holistic_tracker.process(raw_image) # <-this is where the magic happens+ annotated_image = None+ if annotate_image:+ annotated_image = self._annotate_image(raw_image, mediapipe_results)++ return MediapipeSkeletonPayload(mediapipe_results=mediapipe_results,+ annotated_image=annotated_image)++ def _annotate_image(self, image, mediapipe_results):+ self._mp_drawing.draw_landmarks(image=image,+ landmark_list=mediapipe_results.face_landmarks,+ connections=self._mp_holistic.FACEMESH_CONTOURS,+ landmark_drawing_spec=None,+ connection_drawing_spec=self._mp_drawing_styles+ .get_default_face_mesh_contours_style())+ self._mp_drawing.draw_landmarks(image=image,+ landmark_list=mediapipe_results.face_landmarks,+ connections=self._mp_holistic.FACEMESH_TESSELATION,+ landmark_drawing_spec=None,+ connection_drawing_spec=self._mp_drawing_styles+ .get_default_face_mesh_tesselation_style())+ self._mp_drawing.draw_landmarks(image=image,+ landmark_list=mediapipe_results.pose_landmarks,+ connections=self._mp_holistic.POSE_CONNECTIONS,+ landmark_drawing_spec=self._mp_drawing_styles+ .get_default_pose_landmarks_style())+ self._mp_drawing.draw_landmarks(+ image=image,+ landmark_list=mediapipe_results.left_hand_landmarks,+ connections=self._mp_holistic.HAND_CONNECTIONS,+ landmark_drawing_spec=None,+ connection_drawing_spec=self._mp_drawing_styles+ .get_default_hand_connections_style())++ self._mp_drawing.draw_landmarks(+ image=image,+ landmark_list=mediapipe_results.right_hand_landmarks,+ connections=self._mp_holistic.HAND_CONNECTIONS,+ landmark_drawing_spec=None,+ connection_drawing_spec=self._mp_drawing_styles+ .get_default_hand_connections_style())+ return image++ def process_session_folder(self,+ save_annotated_videos:bool=True):+ synchronized_videos_path = Path(get_synchronized_videos_path(self._session_id))++ for video_number, this_synchronized_video_file_path in enumerate(synchronized_videos_path.glob('*.mp4')):+ this_video_capture_object = cv2.VideoCapture(str(this_synchronized_video_file_path))+ logger.info(f"Running `mediapipe` skeleton detection on video: {str(this_synchronized_video_file_path)}")+ this_video_mediapipe_payload_list = []++ success=True+ while success:+ success, image = this_video_capture_object.read()+ if image is None:+ logger.error(f"Failed to load an image from: {str(this_synchronized_video_file_path)}")+ raise Exception+ mediapipe_skeleton_payload = self.detect_skeleton_in_image(image, annotate_image=True)+ this_video_mediapipe_payload_list.append(mediapipe_skeleton_payload)++ if save_annotated_videos:+ self.save_annotated_videos(this_video_mediapipe_payload_list, this_video_capture_object, this_synchronized_video_file_path.stem)++ def save_annotated_videos(self, mediapipe_payload_list:List, video_capture_object:cv2.VideoCapture, video_file_name:str):++ this_video_name = video_file_name + "_mediapipe.mp4"+ image_width = video_capture_object.get(cv2.CAP_PROP_FRAME_WIDTH)+ image_height= video_capture_object.get(cv2.CAP_PROP_FRAME_HEIGHT)++ video_recorder = VideoRecorder(this_video_name,+ image_width,+ image_height,+ self._session_id,+ mediapipe_annotated_video_bool=True)++ image_list = [this_mediapipe_payload.annotated_image for this_mediapipe_payload in mediapipe_payload_list]+ logger.info(f'Saving mediapipe annotated video: {video_recorder.path_to_save_video_file}')+ video_recorder.save_image_list_to_disk(image_list, frame_rate=30)+++++class MediapipeSkeletonPayload(TweakedModel):+ mediapipe_results_list = List+ annotated_image = FramePayloaddiff --git a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.pyindex b432b6d1..3c99868a 100644--- a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py+++ b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py@@ -22,14 +22,19 @@ logger = logging.getLogger(__name__)class CalibrationPipelineOrchestrator:- def __init__(self, session_id:str=None):+ def __init__(self,+ session_id: str = None,+ expected_framerate: Union[int, float] = None,+ ):if session_id is not None:self._session_id = session_idself._calibration_start_time_unix_ns = time.time_ns()self._charuco_board_detector = CharucoBoardDetector()self._visualizer_gui = QTVisualizerAndGui(source='calibration')- self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id)+ self._expected_framerate = expected_framerate+ self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id,+ expected_framerate=self._expected_framerate)@propertydef session_id(self):@@ -48,25 +53,27 @@ class CalibrationPipelineOrchestrator:):"""open all cameras and start recording, detect charuco boards until the user closes the windows, then send recorded videos to `aniposelib` for processingproduces `[session_id]_calibration.toml` file that can be used to estiamte 3d trajectorires from synchronized images"""- with self._open_cv_camera_manager.start_capture_session_all_cams(calibration_videos=True) as connected_cameras_dict:+ with self._open_cv_camera_manager.start_capture_session_all_cams(+ calibration_videos=True,+ ) as connected_cameras_dict:+ timestamp_manager = self._open_cv_camera_manager.timestamp_managertry:if show_visualizer_gui:self._visualizer_gui.setup_and_launch(self._open_cv_camera_manager.available_webcam_ids)- timestamp_manager = TimestampManager(self._open_cv_camera_manager.available_webcam_ids,- self._calibration_start_time_unix_ns)-should_continue = Truewhile should_continue:- timestamp_manager.increment_main_loop_timestamp_logger(time.perf_counter_ns())+ timestamp_manager.log_new_timestamp_for_main_loop_ns(time.perf_counter_ns())- for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():+ if not self._open_cv_camera_manager.new_multi_frame_ready():+ continue- if not this_open_cv_camera.new_frame_ready:- continue+ this_multi_frame_payload = self._open_cv_camera_manager.latest_multi_frame++ for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():- this_cam_latest_frame = this_open_cv_camera.latest_frame+ this_cam_latest_frame = this_multi_frame_payload.frames_dict[this_webcam_id]if this_cam_latest_frame is None:continue@@ -74,17 +81,17 @@ class CalibrationPipelineOrchestrator:# log timestampthis_cam_this_frame_timestamp_ns = this_cam_latest_frame.timestampthis_cam_this_frame_number = this_cam_latest_frame.frame_number- timestamp_manager.increment_frame_processed_for_webcam(this_webcam_id,- this_cam_this_frame_timestamp_ns,- this_cam_this_frame_number)+ timestamp_manager.log_new_timestamp_for_webcam_ns(this_webcam_id,+ this_cam_this_frame_timestamp_ns,+ this_cam_this_frame_number)# save frame to video fileif save_video_in_frame_loop:# save this frame straight to the video file (no risk of memory overflow, but can't handle higher numbers of cameras)- this_open_cv_camera.video_recorder.save_frame_to_video_file(this_cam_latest_frame)+ this_open_cv_camera.video_recorder.save_frame_payload_to_video_file(this_cam_latest_frame)else:# can handle large numbers of cameras, but will eventually fill up RAM and cause a crash- this_open_cv_camera.video_recorder.append_frame_to_list(this_cam_latest_frame)+ this_open_cv_camera.video_recorder.append_frame_payload_to_list(this_cam_latest_frame)# detect charuco boardthis_charuco_frame_payload = self._charuco_board_detector.detect_charuco_board(@@ -119,7 +126,7 @@ class CalibrationPipelineOrchestrator:cv2.destroyAllWindows()for this_open_cv_camera in connected_cameras_dict.values():if not save_video_in_frame_loop:- this_open_cv_camera.video_recorder.save_frame_list_to_disk()+ this_open_cv_camera.video_recorder.save_frame_payload_list_to_disk()this_open_cv_camera.video_recorder.close()if show_visualizer_gui:@@ -127,7 +134,7 @@ class CalibrationPipelineOrchestrator:def run_anipose_camera_calibration(self,charuco_square_size: Union[int, float] = 1,- pin_camera_0_to_origin:bool = False,+ pin_camera_0_to_origin: bool = False,):anipose_camera_calibrator = AniposeCameraCalibrator(self.session_id,self._charuco_board_detector.charuco_board_data_class_object,diff --git a/src/pipelines/session_pipeline/data_classes/multi_frame_payload.py b/src/pipelines/session_pipeline/data_classes/multi_frame_payload.pynew file mode 100644index 00000000..ac741eae--- /dev/null+++ b/src/pipelines/session_pipeline/data_classes/multi_frame_payload.py@@ -0,0 +1,13 @@+from typing import NamedTuple, Union, Dict, List++import numpy as np++from src.cameras.capture.dataclasses.frame_payload import FramePayload+++class MultiFramePayload(NamedTuple):+ frames_dict: Dict[str, FramePayload]+ multi_frame_number: int+ intra_frame_interval: Union[int, float]+ each_frame_timestamp: List[Union[int, float]]+ multi_frame_timestamp: Union[int, float]diff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.pyindex 89e61da0..6aa3dcf2 100644--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py@@ -10,14 +10,15 @@ import numpy as npfrom src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManagerfrom src.cameras.persistence.video_writer.save_options_dataclass import SaveOptionsfrom src.config.data_paths import freemocap_data_path-from src.config.home_dir import create_session_id+from src.config.home_dir import create_session_id, get_session_pathfrom src.core_processor.camera_calibration.camera_calibrator import CameraCalibratorfrom src.core_processor.fps.timestamp_manager import TimestampManager-from src.core_processor.mediapipe_skeleton_detection.mediapipe_skeleton_detection import MediaPipeSkeletonDetection+from src.core_processor.mediapipe_skeleton_detector.mediapipe_skeleton_detector import MediaPipeSkeletonDetectorfrom src.core_processor.show_cam_window import show_cam_windowfrom src.core_processor.utils.image_fps_writer import write_fps_to_imagefrom src.debug_plot_makers.simple_plot_3d_points import simple_plot_3dfrom src.pipelines.calibration_pipeline.calibration_pipeline_orchestrator import CalibrationPipelineOrchestrator+from src.pipelines.calibration_pipeline.charuco_board_detection.charuco_board_detector import CharucoBoardDetectorfrom src.pipelines.session_pipeline.data_classes.data_3d_single_frame_payload import Data3dSingleFramePayloadfrom src.qt_visualizer_and_gui.qt_visualizer_and_gui import QTVisualizerAndGui@@ -31,30 +32,37 @@ def get_canonical_time_str():class SessionPipelineOrchestrator:- def __init__(self, session_id: str = None):+ def __init__(self, session_id: str = None, expected_framerate: Union[int, float] = None):if session_id is not None:self._session_id = session_idelse:self._session_id = create_session_id()- self._session_start_time_unix_ns = time.time_ns()- self._visualizer_gui = QTVisualizerAndGui()- self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id)- self._camera_calibrator = CameraCalibrator()- self._charuco_board_detector = self._camera_calibrator.charuco_board_detector- self._mediapipe_skeleton_detector = MediaPipeSkeletonDetection(model_complexity=2)+ self._expected_framerate = expected_framerate- self._multi_frame_timestamp_intervals_sec = []+ self._visualizer_gui = QTVisualizerAndGui()+ self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id,+ expected_framerate=self.expected_framerate)+ self._charuco_board_detector = CharucoBoardDetector()+ self._mediapipe_skeleton_detector = MediaPipeSkeletonDetector(self._session_id)@propertydef session_id(self):return self._session_id+ @property+ def expected_framerate(self):+ return self._expected_framerate+@propertydef session_folder_path(self):return freemocap_data_path / self._session_id+ @property+ def mediapipe_skeleton_detector(self):+ return self._mediapipe_skeleton_detector+async def process_by_cam_id(self, webcam_id: str, cb):with self._open_cv_camera_manager.start_capture_session_single_cam(webcam_id=webcam_id,@@ -73,7 +81,7 @@ class SessionPipelineOrchestrator:charuco_frame_payload = self._camera_calibrator.calibrate(cv_cam)if cb and charuco_frame_payload.annotated_image is not None:- writer.append_frame_to_list(charuco_frame_payload.raw_frame_payload)+ writer.append_frame_payload_to_list(charuco_frame_payload.raw_frame_payload)await cb(charuco_frame_payload.annotated_image)except Exception as e:logger.error("Printing traceback")@@ -91,7 +99,7 @@ class SessionPipelineOrchestrator:frame_width=cv_cam.image_width(),frame_height=cv_cam.image_height(),)- writer.save_frame_list_to_disk(options)+ writer.save_frame_payload_list_to_disk(options)def calibrate_camera_capture_volume(self, use_most_recent_calibration: bool = False,load_calibration_from_session_id: str = None,@@ -104,7 +112,8 @@ class SessionPipelineOrchestrator:elif load_calibration_from_session_id is not None:calibration_orchestrator = CalibrationPipelineOrchestrator(load_calibration_from_session_id)else: # create new calibration- calibration_orchestrator = CalibrationPipelineOrchestrator(self.session_id)+ calibration_orchestrator = CalibrationPipelineOrchestrator(self.session_id,+ expected_framerate=self.expected_framerate)calibration_orchestrator.record_videos(show_visualizer_gui=False,save_video_in_frame_loop=False,show_camera_views_in_windows=True,@@ -114,56 +123,37 @@ class SessionPipelineOrchestrator:charuco_square_size=charuco_square_size,pin_camera_0_to_origin=pin_camera_0_to_origin)- def run_big_old_bongo_frame_loop(+ def run_main_frame_loop(self,- show_camera_views_in_windows=True,show_visualizer_gui=True,- detect_mediapipe=True,- detect_charuco=True,- reconstruct_3d=True,- save_video=True,+ show_camera_views_in_windows=False,+ detect_mediapipe=False,+ detect_charuco=False,):- self._multi_frames_processed = 0with self._open_cv_camera_manager.start_capture_session_all_cams() as connected_cameras_dict:- self._number_of_cameras = len(connected_cameras_dict)- if detect_charuco:- incoming_charuco_frame_data_per_camera_dict = {}- for this_webcam_id in connected_cameras_dict.keys():- incoming_charuco_frame_data_per_camera_dict[this_webcam_id] = []++ timestamp_manager = self._open_cv_camera_manager.timestamp_managertry:if show_visualizer_gui:self._visualizer_gui.setup_and_launch(self._open_cv_camera_manager.available_webcam_ids)- if detect_charuco:- self._visualizer_gui.initialize_charuco_dottos(- self._charuco_board_detector.number_of_charuco_corners)-- timestamp_manager = TimestampManager(self._open_cv_camera_manager.available_webcam_ids,- self._session_start_time_unix_ns)should_continue = Truewhile should_continue: # BIG FRAME LOOP STARTS HERE- timestamp_manager.increment_main_loop_timestamp_logger(time.perf_counter_ns())+ timestamp_manager.log_new_timestamp_for_main_loop_ns(time.perf_counter_ns())- # reconstruct 3d data, if there's enough new data to so- if reconstruct_3d:- if detect_charuco:- charuco_frame_payload = self._reconstruct_3d_charuco(- incoming_charuco_frame_data_per_camera_dict)+ if not self._open_cv_camera_manager.new_multi_frame_ready():+ continue+ this_multi_frame_payload = self._open_cv_camera_manager.latest_multi_frame- if charuco_frame_payload is not None and False:- simple_plot_3d(charuco_frame_payload.data3d_trackedPointNum_xyz)+ if this_multi_frame_payload is None:+ continue- if show_visualizer_gui and charuco_frame_payload is not None:- self._visualizer_gui.update_charuco_3d_dottos(charuco_frame_payload)for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():- if not this_open_cv_camera.new_frame_ready:- continue-- this_cam_latest_frame = this_open_cv_camera.latest_frame+ this_cam_latest_frame = this_multi_frame_payload.frames_dict[this_webcam_id]if this_cam_latest_frame is None:continue@@ -171,11 +161,18 @@ class SessionPipelineOrchestrator:image_to_display = this_cam_latest_frame.image.copy()this_cam_this_frame_timestamp_ns = this_cam_latest_frame.timestamp- timestamp_manager.increment_frame_processed_for_webcam(this_webcam_id,- this_cam_this_frame_timestamp_ns)+ # log timestamp+ this_cam_this_frame_timestamp_ns = this_cam_latest_frame.timestamp+ this_cam_this_frame_number = this_cam_latest_frame.frame_number+ timestamp_manager.log_new_timestamp_for_webcam_ns(this_webcam_id,+ this_cam_this_frame_timestamp_ns,+ this_cam_this_frame_number)++ # # save this frame straight to the video file (no risk of memory overflow, but can't handle higher numbers of cameras)+ # this_open_cv_camera.video_recorder.save_frame_to_video_file(this_cam_latest_frame)- if save_video:- this_open_cv_camera.video_recorder.append_frame_to_list(this_cam_latest_frame)+ # can handle large numbers of cameras, but will eventually fill up RAM and cause a crash+ this_open_cv_camera.video_recorder.append_frame_payload_to_list(this_cam_latest_frame)if detect_mediapipe:image_to_display = self._mediapipe_skeleton_detector.detect_skeleton_in_image(@@ -184,9 +181,6 @@ class SessionPipelineOrchestrator:if detect_charuco:charuco_frame_payload = self._charuco_board_detector.detect_charuco_board(this_cam_latest_frame)- incoming_charuco_frame_data_per_camera_dict[this_webcam_id].append(- charuco_frame_payload)-image_to_display = charuco_frame_payload.annotated_imageif show_camera_views_in_windows:@@ -202,24 +196,28 @@ class SessionPipelineOrchestrator:self._visualizer_gui.update_camera_view_image(this_webcam_id, image_to_display)# self._visualizer_gui.update_timestamp_plots(timestamp_manager)+ if self._visualizer_gui.shut_it_down:+ logger.info("GUI closed.")+ should_continue = False+# exit loop when user presses ESC keyexit_key = cv2.waitKey(1)if exit_key == 27:logger.info("ESC has been pressed.")should_continue = False-+except:logger.error("Printing traceback")traceback.print_exc()finally:for this_open_cv_camera in connected_cameras_dict.values():- if save_video:- this_open_cv_camera.video_recorder.save_frame_list_to_disk(- self.session_folder_path / 'synchronized_videos')- logger.info(f"Destroy window {this_open_cv_camera.webcam_id_as_str}")- cv2.destroyWindow(this_open_cv_camera.webcam_id_as_str)- cv2.waitKey(1)++ this_open_cv_camera.video_recorder.save_frame_payload_list_to_disk()++ if show_camera_views_in_windows:+ logger.info(f"Destroy window {this_open_cv_camera.webcam_id_as_str}")+ cv2.destroyWindow(this_open_cv_camera.webcam_id_as_str)if show_visualizer_gui:self._visualizer_gui.close()@@ -251,23 +249,13 @@ class SessionPipelineOrchestrator:this_multi_frame_charuco_data.append(this_cam_this_frame)is_there_any_charuco_data.append(this_cam_this_frame.charuco_view_data.some_charuco_corners_found)- timestamp_min = np.min(this_multi_frame_timestamps)- timestamp_max = np.max(this_multi_frame_timestamps)- this_multiframe_timestamp_interval_sec = (timestamp_max - timestamp_min) / 1e9- #TODO - should bring the `TimeStampManager` into this?- self._multi_frame_timestamp_intervals_sec.append(this_multiframe_timestamp_interval_sec)-- print(f"TODO:sync with timestamps, not frame index - multi_frame#: {self._multi_frames_processed - 1},"- f" each_cam_#_of_frames: {each_cam_number_of_frames}, "- f" intra_multi_frame_timestamp_range: {this_multiframe_timestamp_interval_sec:.3f}", )-min_cameras_to_reconstruct = 2if sum(is_there_any_charuco_data) >= min_cameras_to_reconstruct: # if at least 2 cameras have any data, then we can try to make some 3d dottoscharuco2d_data_per_cam_dict = self.format_charuco2d_data(this_multi_frame_charuco_data)charuco_3d_data_payload = self._triangulate_2d_data(charuco2d_data_per_cam_dict,self._charuco_board_detector.number_of_charuco_corners)- x = charuco_3d_data_payload.data3d_trackedPointNum_xyz[:,0]+ x = charuco_3d_data_payload.data3d_trackedPointNum_xyz[:, 0]return charuco_3d_data_payloaddef format_charuco2d_data(self, this_multi_frame_charuco_data) -> Dict:@@ -298,8 +286,8 @@ class SessionPipelineOrchestrator:data2d_trackedPointNum_xyz_camNum = np.dstack(each_cam2d_data_list) # stack lists (in depth) to make one numpy array with dimensions [number_of_tracked_points, XYZ, number_of_cameras]data2d_camNum_trackedPointNum_xy = data2d_trackedPointNum_xyz_camNum.reshape(self._number_of_cameras,- number_of_tracked_points,- 2) # reshape to fit into anipose (new dimensions [number_of_cameras, number_of_tracked_points, XYZ]+ number_of_tracked_points,+ 2) # reshape to fit into anipose (new dimensions [number_of_cameras, number_of_tracked_points, XYZ]# THIS IS WHERE THE MAGIC HAPPENS - 2d data from calibrated, synchronized cameras has now become a 3d estimate. Hurray! :`Ddata3d_trackedPointNum_xyz = self._anipose_camera_calibration_object.triangulate(@@ -323,18 +311,19 @@ if __name__ == "__main__":print('running `session_pipeline_orchestrator` as `__main__')length_of_one_edge_of_a_black_square_on_the_charuco_board_in_mm = 39+ expected_framerate = 30++ calibrate_cameras = False++ this_session_orchestrator = SessionPipelineOrchestrator(expected_framerate=expected_framerate)- this_session_orchestrator = SessionPipelineOrchestrator()- # this_session_orchestrator.calibrate_camera_capture_volume(use_most_recent_calibration=True)+ if calibrate_cameras:+ this_session_orchestrator.calibrate_camera_capture_volume(+ charuco_square_size=length_of_one_edge_of_a_black_square_on_the_charuco_board_in_mm,+ pin_camera_0_to_origin=True)- this_session_orchestrator.calibrate_camera_capture_volume(- charuco_square_size=length_of_one_edge_of_a_black_square_on_the_charuco_board_in_mm,- pin_camera_0_to_origin=True)+ this_session_orchestrator.run_main_frame_loop(show_camera_views_in_windows=False,+ show_visualizer_gui=True,+ )- this_session_orchestrator.run_big_old_bongo_frame_loop(show_camera_views_in_windows=False,- save_video=False,- show_visualizer_gui=True,- detect_mediapipe=False,- detect_charuco=True,- reconstruct_3d=True,- )+ this_session_orchestrator.mediapipe_skeleton_detector.process_session_folder(save_annotated_videos=True)diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.pyindex 852aefa3..3ecd64d6 100644--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py@@ -19,9 +19,8 @@ logger = logging.getLogger(__name__)class QTVisualizerAndGui:def __init__(self, source:str=None):# https://pyqtgraph.readthedocs.io/en/latest/config_options.html- self._camera0_id = '0'+ self.pyqtgraph_app = pg.mkQApp('freemocap! :D')pg.setConfigOptions(imageAxisOrder='row-major')- self.pyqtgraph_app = pg.mkQApp("FreeMoCap! :O ")self._dict_of_camera_image_item_widgets = {}self._dict_of_simple_timestamp_line_plots = {}@@ -33,14 +32,24 @@ class QTVisualizerAndGui:self._webcam_ids_list = []self._is_paused = False+ self._shut_it_down = False++ @property+ def shut_it_down(self):+ return self._shut_it_down@propertydef pause_button_pressed(self):return self._is_paused+ def _close_button_pressed(self):+ self.close()+ self._shut_it_down = True+def close(self):self._main_window_widget.close()+def setup_and_launch(self, webcam_ids_list):logger.info('setting up QT Visualizer and GUI')self._number_of_cameras = len(webcam_ids_list)@@ -79,16 +88,16 @@ class QTVisualizerAndGui:def _update_simple_timestamp_plot(self, timestamp_manager: TimestampManager):for webcam_id in self._webcam_ids_list:plot_item = self._dict_of_simple_timestamp_line_plots[webcam_id]- plot_item.setData(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps)+ plot_item.setData(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps_unix_ns)def _update_timestamp_difference_plot(self, timestamp_manager: TimestampManager):for webcam_id in self._webcam_ids_list:plot_item = self._dict_of_simple_timestamp_line_plots[webcam_id]- plot_item.setData(np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps))+ plot_item.setData(np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps_unix_ns))def _update_timestamp_difference_histogram(self, timestamp_manager: TimestampManager):for webcam_id in self._webcam_ids_list:- this_cam_timestamp_diffs = np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps)+ this_cam_timestamp_diffs = np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps_unix_ns)num_samples = this_cam_timestamp_diffs.shape[0]counts, bin_edges = np.histogram(this_cam_timestamp_diffs,bins=np.linspace(0, 100, 100))@@ -131,15 +140,20 @@ class QTVisualizerAndGui:self._play_button.clicked.connect(self._play)control_panel_layout_widget.addWidget(self._play_button, row=2, col=0)- self._reset_calibration_button = QtWidgets.QPushButton('Reset Calibration')- self._reset_calibration_button.setEnabled(False)- self._reset_calibration_button.clicked.connect(self._reset_calibration)- control_panel_layout_widget.addWidget(self._reset_calibration_button, row=3, col=0)-- self._record_button = QtWidgets.QPushButton('Record')- self._record_button.setEnabled(False)- self._record_button.clicked.connect(self._record)- control_panel_layout_widget.addWidget(self._record_button, row=3, col=0)+ # self._reset_calibration_button = QtWidgets.QPushButton('Reset Calibration')+ # self._reset_calibration_button.setEnabled(False)+ # self._reset_calibration_button.clicked.connect(self._reset_calibration)+ # control_panel_layout_widget.addWidget(self._reset_calibration_button, row=3, col=0)+ #+ # self._record_button = QtWidgets.QPushButton('Record')+ # self._record_button.setEnabled(False)+ # self._record_button.clicked.connect(self._record)+ # control_panel_layout_widget.addWidget(self._record_button, row=3, col=0)++ self._close_button = QtWidgets.QPushButton('Close All')+ self._close_button.setEnabled(True)+ self._close_button.clicked.connect(self._close_button_pressed)+ control_panel_layout_widget.addWidget(self._close_button, row=4, col=0)self._main_dock_area.addDock(self._control_panel_dock, position='left')diff --git a/src/tests/video_writer/test_video_writer.py b/src/tests/video_writer/test_video_writer.pyindex f954ed71..97ba6fc3 100644--- a/src/tests/video_writer/test_video_writer.py+++ b/src/tests/video_writer/test_video_writer.py@@ -23,7 +23,7 @@ class VideoWriterTestCase(TestCase):image=np.random.randint(0, 4, (720, 1280, 3)), timestamp=time.time_ns())vw = VideoRecorder()- vw.append_frame_to_list(example_payload)+ vw.append_frame_payload_to_list(example_payload)assert vw.frame_count == 1@@ -33,7 +33,7 @@ class VideoWriterTestCase(TestCase):image=np.random.randint(0, 4, (720, 1280, 3)), timestamp=time.time_ns())vw = VideoRecorder()- vw.append_frame_to_list(example_payload)+ vw.append_frame_payload_to_list(example_payload)save_options = SaveOptions(writer_dir=file_path,@@ -43,7 +43,7 @@ class VideoWriterTestCase(TestCase):fps=1,)- vw.save_frame_list_to_disk(save_options)+ vw.save_frame_payload_list_to_disk(save_options)expected_path = Path().joinpath(file_path, "movie.mp4")assert save_options.full_path == expected_path, "Paths do not match"