Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

some kinda progress

Jon2baaff176526
diff --git a/requirements.txt b/requirements.txt
index e5286eca..d7ebd8e2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -34,4 +34,5 @@ pyqt5==5.15.6
pyopengl==3.1.6
mediapipe~=0.8.9.1
setuptools~=58.0.4
-aniposelib~=0.4.3
\ No newline at end of file
+aniposelib~=0.4.3
+rich~=12.3.0
\ No newline at end of file
diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.py
index 9577557b..90ed3ba7 100644
--- a/src/cameras/capture/opencv_camera/opencv_camera.py
+++ b/src/cameras/capture/opencv_camera/opencv_camera.py
@@ -115,16 +115,13 @@ class OpenCVCamera:
else:
image_height = image.shape[0]
image_width = image.shape[1]
- try:
- self._video_recorder = VideoRecorder(self._name,
- image_width=image_width,
- image_height=image_height,
- session_id=self.session_id,
- calibration_video_bool=self._calibration_video_bool)
- except:
- logger.error(
- f"could not create video recorder for image with shape (image_width={image_width}, image_height={image_height}) ")
- return False # NOTE- this is a failure because we *tried* to make a video_recorder and it didn't work
+
+ self._video_recorder = VideoRecorder(self._name,
+ image_width=image_width,
+ image_height=image_height,
+ session_id=self.session_id,
+ calibration_video_bool=self._calibration_video_bool)
+
return success
diff --git a/src/cameras/multicam_manager/cv_camera_manager.py b/src/cameras/multicam_manager/cv_camera_manager.py
index 426f1bef..d0e7732d 100644
--- a/src/cameras/multicam_manager/cv_camera_manager.py
+++ b/src/cameras/multicam_manager/cv_camera_manager.py
@@ -1,8 +1,10 @@
import logging
+import time
import traceback
from contextlib import contextmanager
-from typing import ContextManager, Dict, List, Optional
+from typing import ContextManager, Dict, List, Optional, Union
+import numpy as np
from pydantic import BaseModel
from src.api.services.user_config import UserConfigService
@@ -10,6 +12,8 @@ from src.cameras.capture.opencv_camera.opencv_camera import OpenCVCamera
from src.cameras.detection.cam_singleton import get_or_create_cams
from src.cameras.persistence.video_writer.video_recorder import VideoRecorder
from src.config.webcam_config import WebcamConfig
+from src.core_processor.fps.timestamp_manager import TimestampManager
+from src.pipelines.session_pipeline.data_classes.multi_frame_payload import MultiFramePayload
logger = logging.getLogger(__name__)
@@ -23,44 +27,79 @@ class CamAndWriterResponse(BaseModel):
class OpenCVCameraManager:
- def __init__(self, session_id: str = None):
+ def __init__(self,
+ session_id: str = None,
+ expected_framerate: Union[int, float] = None,
+ ):
self._session_id = session_id
self._config_service = UserConfigService()
self._detected_cams_data = get_or_create_cams()
+ self._session_start_time_unix_ns = None
+ self._timestamp_manager: Union[TimestampManager, None] = None
+ self._connected_cameras_dict = {}
+ self._expected_framerate = expected_framerate
+
+ @property
+ def timestamp_manager(self):
+ if self._timestamp_manager is None:
+ logger.error('timestamp manager has not been created yet')
+ else:
+ return self._timestamp_manager
@property
def available_webcam_ids(self):
return [cam_id.webcam_id for cam_id in get_or_create_cams().cameras_found_list]
- def _create_opencv_cameras(self, calibration_videos:bool=False):
- raw_camera_objects = self._detected_cams_data.cameras_found_list
- open_cv_cameras: List[OpenCVCamera] = []
- for this_raw_cam in raw_camera_objects:
- opencv_cam_obj = self._create_single_opencv_cam(this_raw_cam.webcam_id, calibration_video_bool =calibration_videos)
- open_cv_cameras.append(opencv_cam_obj)
- return open_cv_cameras
-
- def _create_single_opencv_cam(self, webcam_id: str, calibration_video_bool:bool =False):
- webcam_config_model = self._config_service.webcam_config_by_id(webcam_id, self._session_id)
- single_camera_config = WebcamConfig(webcam_id=webcam_config_model.webcam_id,
- exposure=webcam_config_model.exposure,
- resolution_width=webcam_config_model.resolution_width,
- resolution_height=webcam_config_model.resolution_height, )
- return OpenCVCamera(config=single_camera_config,
- session_id=self._session_id,
- calibration_video_bool = calibration_video_bool)
+ @property
+ def latest_multi_frame(self) -> Union[MultiFramePayload,None]:
+ if not self.new_multi_frame_ready():
+ logging.error("a multi_frame was requested before it was ready!")
+ raise Exception
+
+ this_multi_frame_dict = {}
+ for this_cam in self._connected_cameras_dict.values():
+ if not this_cam.new_frame_ready:
+ logger.error('It shouldnt be able to get here if the `new_multi_frame`')
+ this_multi_frame_dict[this_cam.webcam_id_as_str] = this_cam.latest_frame
+ self._timestamp_manager.log_new_multi_frame_timestamp_ns(time.perf_counter_ns())
+
+ if not self._timestamp_manager.verify_multi_frame_is_synchronized(this_multi_frame_dict,
+ self._expected_framerate):
+ m_f_interval = self._timestamp_manager.latest_multi_frame_interval
+ m_f_number = self._timestamp_manager.multi_frame_timestamp_logger.number_of_frames
+ logger.error(f"Multi frame is not synchronized!! Skipping this one - multi_frame_frame_number|interval: {m_f_number}|{m_f_interval:.3f}")
+ # raise Exception
+ return None
+
+ return MultiFramePayload(frames_dict=this_multi_frame_dict,
+ multi_frame_number=self._timestamp_manager.multi_frame_timestamp_logger.number_of_frames,
+ intra_frame_interval=self._timestamp_manager.latest_multi_frame_interval,
+ each_frame_timestamp=self._timestamp_manager.latest_multi_frame_timestamp_list,
+ multi_frame_timestamp=self._timestamp_manager.multi_frame_timestamp_logger.latest_timestamp,
+ )
+
+ def new_multi_frame_ready(self):
+ """cycle through connected cameras and return false if one isn't read yet
+ TODO - make another kind of check that uses a clock and sends an empty frame if one camera hasn't yeild a frame within a reasonable time period. With this method, the whol camera system will stall if one camera stops producing frames"""
+ for this_cam in self._connected_cameras_dict.values():
+ if not this_cam.new_frame_ready:
+ return False
+ return True
@contextmanager
def start_capture_session_single_cam(
- self, webcam_id: str
+ self, webcam_id: str
) -> ContextManager[CamAndWriterResponse]:
"""
Context manager for easy start up, usage, and cleanup of camera resources.
Can capture frames from a single webcam or all webcams detected.
"""
+
cv_camera = self._create_single_opencv_cam(webcam_id)
+ self._connected_cameras_dict['0'] = cv_camera
try:
self._start_frame_capture_on_cam_id(cv_camera)
+ self._initialize_timestamp_logger() # start timestamp logger and wha
yield CamAndWriterResponse(cv_camera=cv_camera)
self._stop_frame_capture([cv_camera])
except:
@@ -69,23 +108,49 @@ class OpenCVCameraManager:
@contextmanager
def start_capture_session_all_cams(
- self,
+ self,
calibration_videos=False,
) -> ContextManager[Dict[str, OpenCVCamera]]:
-
- open_cv_camera_objects = self._create_opencv_cameras(calibration_videos=True)
+ open_cv_camera_objects = self._create_opencv_cameras(calibration_videos=calibration_videos)
try:
- connected_cameras_dict = {}
+
for cv_cam in open_cv_camera_objects:
- connected_cameras_dict[cv_cam.webcam_id_as_str] = cv_cam
+ self._connected_cameras_dict[cv_cam.webcam_id_as_str] = cv_cam
self._start_frame_capture_on_cam_id(cv_cam)
- yield connected_cameras_dict
+ self._initialize_timestamp_logger()
+ yield self._connected_cameras_dict
self._stop_frame_capture(open_cv_camera_objects)
except:
logger.error("Printing traceback from starting capture session by cam")
traceback.print_exc()
+ def _create_opencv_cameras(self, calibration_videos: bool = False):
+ raw_camera_objects = self._detected_cams_data.cameras_found_list
+ open_cv_cameras: List[OpenCVCamera] = []
+ for this_raw_cam in raw_camera_objects:
+ opencv_cam_obj = self._create_single_opencv_cam(this_raw_cam.webcam_id,
+ calibration_video_bool=calibration_videos)
+ open_cv_cameras.append(opencv_cam_obj)
+ return open_cv_cameras
+
+ def _create_single_opencv_cam(self, webcam_id: str, calibration_video_bool: bool = False):
+ webcam_config_model = self._config_service.webcam_config_by_id(webcam_id, self._session_id)
+ single_camera_config = WebcamConfig(webcam_id=webcam_config_model.webcam_id,
+ exposure=webcam_config_model.exposure,
+ resolution_width=webcam_config_model.resolution_width,
+ resolution_height=webcam_config_model.resolution_height, )
+ return OpenCVCamera(config=single_camera_config,
+ session_id=self._session_id,
+ calibration_video_bool=calibration_video_bool)
+
+ def _initialize_timestamp_logger(self):
+ self._session_start_time_unix_ns = time.time_ns()
+ self._session_start_time_perf_counter_ns = time.perf_counter_ns()
+ self._timestamp_manager = TimestampManager(self.available_webcam_ids,
+ self._session_start_time_unix_ns,
+ self._session_start_time_perf_counter_ns)
+
def _start_frame_capture_on_cam_id(self, opencv_cam: OpenCVCamera):
opencv_cam.connect()
opencv_cam.start_frame_capture_thread()
diff --git a/src/cameras/persistence/video_writer/video_recorder.py b/src/cameras/persistence/video_writer/video_recorder.py
index 34ccb177..f887aff0 100644
--- a/src/cameras/persistence/video_writer/video_recorder.py
+++ b/src/cameras/persistence/video_writer/video_recorder.py
@@ -2,7 +2,7 @@ import logging
import os
import traceback
from pathlib import Path
-from typing import List
+from typing import List, Union
import cv2
import numpy as np
@@ -11,44 +11,53 @@ import pandas as pd
from src.cameras.capture.dataclasses.frame_payload import FramePayload
from src.cameras.persistence.video_writer.save_options_dataclass import SaveOptions
from src.config.data_paths import freemocap_data_path
-from src.config.home_dir import get_session_path
+from src.config.home_dir import get_session_path, get_synchronized_videos_path, get_calibration_videos_path, \
+ get_mediapipe_annotated_videos_path
+from rich.progress import Progress
logger = logging.getLogger(__name__)
class VideoRecorder:
def __init__(self,
- camera_name: str,
+ video_name: str,
image_width: int,
image_height: int,
session_id: str,
fourcc: str = "MP4V",
- calibration_video_bool:bool=False,
+ calibration_video_bool: bool = False,
+ mediapipe_annotated_video_bool: bool = False,
):
- self._camera_name = camera_name
+ self._video_name = video_name
self._image_width = image_width
self._image_height = image_height
self._fourcc = fourcc
- self._frame_list: List[FramePayload] = []
- self._video_type_str = ''
+ self._frame_payload_list: List[FramePayload] = []
self._timestamps_npy = np.empty(0)
self._cv2_video_writer = None
# get yr paths straight
- session_path = Path(get_session_path(session_id))
- if calibration_video_bool:
- self._synchronized_videos_folder_path = session_path / 'calibration_videos'
+ self._session_id = session_id
+ session_path = Path(get_session_path(self._session_id))
+
+ if mediapipe_annotated_video_bool:
+ self._video_folder_path = Path(get_mediapipe_annotated_videos_path(self._session_id))
+ elif calibration_video_bool:
+ self._video_folder_path = Path(get_calibration_videos_path(self._session_id))
else:
- self._synchronized_videos_folder_path = session_path / 'synchronized_videos'
+ self._video_folder_path = Path(get_synchronized_videos_path(self._session_id))
- self._synchronized_videos_folder_path.mkdir(parents=True, exist_ok=True)
- video_file_name = self._camera_name + '.mp4'
- self._path_to_save_video_file = self._synchronized_videos_folder_path / video_file_name
+ self._video_folder_path.mkdir(parents=True, exist_ok=True)
+ video_file_name = self._video_name + '.mp4'
+ self._path_to_save_video_file = self._video_folder_path / video_file_name
+ @property
+ def video_name(self):
+ return self._video_name
@property
- def synchronized_videos_folder_path(self):
- return self._synchronized_videos_folder_path
+ def video_folder_path(self):
+ return self._video_folder_path
@property
def path_to_save_video_file(self):
@@ -56,20 +65,20 @@ class VideoRecorder:
@property
def frame_count(self):
- return len(self._frame_list)
+ return len(self._frame_payload_list)
@property
def median_framerate(self):
- if self.frame_count < 10:
- logger.warning(f"camera framerate not known, setting video playback rate to 30 frames per second")
- self._median_framerate = 30
+ if self.frame_count == 0:
+ logger.error(f"No Frames to save for {self.path_to_save_video_file}")
+ raise Exception
else:
self._gather_timestamps()
self._median_framerate = (np.nanmedian(np.diff(self._timestamps_npy / 1e9))) ** -1
return self._median_framerate
- def save_frame_to_video_file(self,frame_payload: FramePayload):
+ def save_frame_payload_to_video_file(self, frame_payload: FramePayload):
if self._cv2_video_writer is None:
self._initialize_video_writer()
@@ -78,12 +87,12 @@ class VideoRecorder:
def close(self):
self._cv2_video_writer.release()
- def append_frame_to_list(self, frame_payload: FramePayload):
- self._frame_list.append(frame_payload)
+ def append_frame_payload_to_list(self, frame_payload: FramePayload):
+ self._frame_payload_list.append(frame_payload)
- def save_frame_list_to_disk(self):
- if len(self._frame_list) == 0:
- logging.error(f"No frames to save for camera: {self._camera_name}")
+ def save_frame_payload_list_to_disk(self):
+ if len(self._frame_payload_list) == 0:
+ logging.error(f"No frames to save for camera: {self._video_name}")
return
self._gather_timestamps()
@@ -91,18 +100,32 @@ class VideoRecorder:
self._write_frame_list_to_video_file()
self._save_timestamps()
- def _initialize_video_writer(self):
+ def save_image_list_to_disk(self,
+ image_list: List[np.ndarray],
+ frame_rate: Union[int, float] = None):
+ if len(image_list) == 0:
+ logging.error(f"No frames to save for : {self._video_name}")
+ return
+
+ self._initialize_video_writer(frames_per_second=frame_rate)
+ self._write_image_list_to_video_file(image_list)
+
+ def _initialize_video_writer(self, frames_per_second: Union[int, float] = None):
+
+ if frames_per_second is None:
+ frames_per_second = self.median_framerate
+ else:
+ frames_per_second = frames_per_second
self._cv2_video_writer = cv2.VideoWriter(
str(self.path_to_save_video_file),
cv2.VideoWriter_fourcc(*self._fourcc),
- self.median_framerate,
- (self._image_width, self._image_height))
+ frames_per_second,
+ (int(self._image_width), int(self._image_height)))
def _write_frame_list_to_video_file(self):
-
try:
- for frame in self._frame_list:
+ for frame in self._frame_payload_list:
self._cv2_video_writer.write(frame.image)
except Exception as e:
@@ -113,21 +136,35 @@ class VideoRecorder:
logger.info(f"Saved video to path: {self.path_to_save_video_file}")
self._cv2_video_writer.release()
+ def _write_image_list_to_video_file(self, image_list:List[np.ndarray]):
+ try:
+ for image in image_list:
+ self._cv2_video_writer.write(image)
+
+
+ except Exception as e:
+ logger.error(f"Failed during save in video writer: {self.path_to_save_video_file}")
+ traceback.print_exc()
+ raise e
+ finally:
+ logger.info(f"Saved video to path: {self.path_to_save_video_file}")
+ self._cv2_video_writer.release()
+
def _gather_timestamps(self):
try:
- for frame in self._frame_list:
+ for frame in self._frame_payload_list:
self._timestamps_npy = np.append(self._timestamps_npy, frame.timestamp)
except:
logger.error("Error gathering timestamps")
def _save_timestamps(self):
- timestamp_file_name_npy = self._camera_name + "_timestamps_binary.npy"
- timestamp_npy_full_save_path = self._synchronized_videos_folder_path / timestamp_file_name_npy
+ timestamp_file_name_npy = self._video_name + "_timestamps_binary.npy"
+ timestamp_npy_full_save_path = self._video_folder_path / timestamp_file_name_npy
np.save(str(timestamp_npy_full_save_path), self._timestamps_npy)
logger.info(f"Saved timestamps to path: {timestamp_file_name_npy}")
- timestamp_file_name_csv = self._camera_name + "_timestamps_human_readable.csv"
- timestamp_csv_full_save_path = self._synchronized_videos_folder_path / timestamp_file_name_csv
+ timestamp_file_name_csv = self._video_name + "_timestamps_human_readable.csv"
+ timestamp_csv_full_save_path = self._video_folder_path / timestamp_file_name_csv
timestamp_dataframe = pd.DataFrame(self._timestamps_npy)
timestamp_dataframe.to_csv(str(timestamp_csv_full_save_path))
logger.info(f"Saved timestamps to path: {timestamp_file_name_csv}")
diff --git a/src/config/home_dir.py b/src/config/home_dir.py
index 5621cd80..fc173639 100644
--- a/src/config/home_dir.py
+++ b/src/config/home_dir.py
@@ -22,3 +22,15 @@ def get_session_path(session_id: str):
base_save_path = Path(get_freemocap_data_folder_path())
session_path = base_save_path / session_id
return str(session_path)
+
+def get_synchronized_videos_path(session_id: str):
+ synchronized_videos_path = Path(get_session_path(session_id)) / 'synchronized_videos'
+ return str(synchronized_videos_path)
+
+def get_calibration_videos_path(session_id: str):
+ calibration_videos_path = Path(get_session_path(session_id)) / 'calibration_videos'
+ return str(calibration_videos_path)
+
+def get_mediapipe_annotated_videos_path(session_id: str):
+ mediapipe_annotated_videos_path = Path(get_session_path(session_id)) / 'mediapipe_annotated_videos'
+ return str(mediapipe_annotated_videos_path)
diff --git a/src/core_processor/fps/timestamp_manager.py b/src/core_processor/fps/timestamp_manager.py
index 04c391a0..e2d3445e 100644
--- a/src/core_processor/fps/timestamp_manager.py
+++ b/src/core_processor/fps/timestamp_manager.py
@@ -1,16 +1,19 @@
+import logging
from typing import Dict, List, Union
import numpy as np
+logger = logging.getLogger(__name__)
+
class TimestampLogger:
_num_frames_processed: int
_start_time: float
- def __init__(self, _session_start_time_unix_ns: int):
- self._session_start_time_unix_ns = _session_start_time_unix_ns
+ def __init__(self, _session_start_time_perf_counter_ns: int):
+ self._session_start_time_perf_counter_ns = _session_start_time_perf_counter_ns
self._num_frames_processed = 0
- self._timestamps = np.empty(0)
+ self._timestamps_ns = np.empty(0)
def is_started(self):
return self._num_frames_processed > 0
@@ -19,41 +22,74 @@ class TimestampLogger:
def median_frames_per_second(self):
if self._num_frames_processed <= 10:
return 0
- return np.nanmedian(np.diff(self._timestamps) / 1e9) ** -1
+ return np.nanmedian(np.diff(self._timestamps_ns) / 1e9) ** -1
@property
def number_of_frames(self):
return self._num_frames_processed
@property
- def timestamps(self):
+ def timestamps_unix_ns(self):
"""return timestamps in nanoseconds in Unix epoch format (which is both the most useful, ugliest, AND funniest time format around. You can use `self.timestamps_in_seconds_from_zero()` for a more intuitive time format"""
- return self._timestamps
+ return self._timestamps_ns - self._session_start_time_perf_counter_ns
+
+ @property
+ def timestamps(self):
+ return self.timestamps_in_seconds_from_session_start
@property
def timestamps_in_seconds_from_session_start(self):
"""timestamps in seconds counting up from zero (where zero is the start time of the recording session)"""
- return (self._timestamps - self._session_start_time_unix_ns) / 1e9
+ return (self._timestamps_ns - self._session_start_time_perf_counter_ns) / 1e9
@property
- def last_timestamp(self):
- if self._timestamps.shape[0] > 0:
- return self._timestamps[-1]
+ def latest_timestamp(self):
+ """returns timestamps in seconds from session start"""
+ if self.timestamps.shape[0] > 0:
+ return self.timestamps[-1]
return None
- def log_new_timestamp(self, timestamp):
- self._timestamps = np.append(self._timestamps, timestamp)
+ def log_new_timestamp_ns(self, timestamp):
+ self._timestamps_ns = np.append(self._timestamps_ns, timestamp)
self._num_frames_processed += 1
class TimestampManager:
- def __init__(self, webcam_ids: List[str], session_start_time_unix_ns: int):
- self._session_start_time_unix_ns = session_start_time_unix_ns
+ def __init__(self,
+ webcam_ids: List[str],
+ session_start_time_unix_ns: int,
+ _session_start_time_perf_counter_ns:int):
+ self._session_start_time_unix_ns = session_start_time_unix_ns #time.time_ns(), UNIX epoch time, nanoseconds
+ self._session_start_time_perf_counter_ns = _session_start_time_perf_counter_ns #time.perf_counter_ns, arbitrary timebase, nanoseconds - corresponds to self.session_start_time_ns
self._webcam_ids = webcam_ids
- self._webcam_timestamp_loggers: Dict[str, TimestampLogger] = self._initialize_timestamp_loggers(webcam_ids)
- self._main_loop_timestamp_logger = TimestampLogger(self._session_start_time_unix_ns)
+ self._webcam_timestamp_loggers: Dict[str, TimestampLogger] = self._initialize_webcam_timestamp_loggers(
+ webcam_ids)
+ self._main_loop_timestamp_logger = TimestampLogger(self._session_start_time_perf_counter_ns)
+ self._multi_frame_timestamp_logger = TimestampLogger(self._session_start_time_perf_counter_ns)
+ self._multi_frame_timestamps_list = []
+ self._multi_frame_interval_list = []
+
+ @property
+ def multi_frame_timestamp_logger(self):
+ return self._multi_frame_timestamp_logger
+
+ @property
+ def multi_frame_timestamps_list(self):
+ return self._multi_frame_timestamps_list
+
+ @property
+ def multi_frame_interval_list(self):
+ return self._multi_frame_interval_list
+
+ @property
+ def latest_multi_frame_timestamp_list(self):
+ return self._multi_frame_timestamps_list[-1]
+
+ @property
+ def latest_multi_frame_interval(self):
+ return self._multi_frame_interval_list[-1]
- def _initialize_timestamp_loggers(self, webcam_ids: List[str]):
+ def _initialize_webcam_timestamp_loggers(self, webcam_ids: List[str]):
dictionary_of_timestamp_logger_objects = {}
for webcam_id in webcam_ids:
dictionary_of_timestamp_logger_objects[webcam_id] = TimestampLogger(self._session_start_time_unix_ns)
@@ -62,20 +98,55 @@ class TimestampManager:
def get_timestamps_from_camera_in_seconds_from_session_start(self, webcam_id):
return self._webcam_timestamp_loggers[webcam_id].timestamps_in_seconds_from_session_start
- def get_number_of_frames(self, webcam_id):
+ def number_of_frames(self, webcam_id):
return self._webcam_timestamp_loggers[webcam_id].number_of_frames
- def increment_main_loop_timestamp_logger(self, timestamp):
- self._main_loop_timestamp_logger.log_new_timestamp(timestamp)
-
def median_frames_per_second_for_main_loop(self):
return self._main_loop_timestamp_logger.median_frames_per_second
- def increment_frame_processed_for_webcam(self, webcam_id: str, timestamp: Union[float, int],
- frame_number: int = None):
- self._webcam_timestamp_loggers[webcam_id].log_new_timestamp(timestamp)
- if frame_number is not None:
- assert (self._webcam_timestamp_loggers[webcam_id].number_of_frames == frame_number, f"Numer of timestamps ({self._webcam_timestamp_loggers[webcam_id].number_of_frames}) logged does not match frame_number ({frame_number}) for: camera_{webcam_id}")
-
def median_frames_per_second_for_webcam(self, webcam_id):
return self._webcam_timestamp_loggers[webcam_id].median_frames_per_second
+
+ def log_new_timestamp_for_main_loop_ns(self, timestamp):
+ self._main_loop_timestamp_logger.log_new_timestamp_ns(timestamp)
+
+ def log_new_multi_frame_timestamp_ns(self, timestamp):
+ self._multi_frame_timestamp_logger.log_new_timestamp_ns(timestamp)
+
+ def log_new_timestamp_for_webcam_ns(self, webcam_id: str, timestamp: Union[float, int],
+ frame_number: int = None):
+ self._webcam_timestamp_loggers[webcam_id].log_new_timestamp_ns(timestamp)
+ if frame_number is not None:
+ assert (self._webcam_timestamp_loggers[webcam_id].number_of_frames == frame_number,
+ f"Numer of timestamps ({self._webcam_timestamp_loggers[webcam_id].number_of_frames}) logged does not match frame_number ({frame_number}) for: camera_{webcam_id}")
+
+ def _get_each_frame_timestamp(self, this_multi_frame_dict) -> List:
+
+ this_multi_frame_timestamps = []
+ for this_cam_id, this_cam_frame in this_multi_frame_dict.items():
+ this_multi_frame_timestamps.append(this_cam_frame.timestamp)
+ return this_multi_frame_timestamps
+
+ def _get_intra_frame_interval_in_sec(self, this_multi_frame_timestamps) -> float:
+ timestamp_min = np.min(this_multi_frame_timestamps)
+ timestamp_max = np.max(this_multi_frame_timestamps)
+ return (timestamp_max - timestamp_min) / 1e9
+
+ def verify_multi_frame_is_synchronized(self,
+ this_multi_frame_dict,
+ expected_framerate: Union[int, None]) -> bool:
+
+ this_multi_frame_timestamps = self._get_each_frame_timestamp(this_multi_frame_dict)
+ this_multiframe_timestamp_interval_sec = self._get_intra_frame_interval_in_sec(this_multi_frame_timestamps)
+
+ self._multi_frame_timestamps_list.append(this_multi_frame_timestamps)
+ self._multi_frame_interval_list.append(this_multiframe_timestamp_interval_sec)
+
+ if expected_framerate is None:
+ logger.warning('`expected_framerate` not specified, Cannot verify multi_frame synchronization')
+ return True
+
+ if this_multiframe_timestamp_interval_sec > 2*(expected_framerate ** -1):
+ return False
+
+ return True
diff --git a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py
deleted file mode 100644
index 69971e30..00000000
--- a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py
+++ /dev/null
@@ -1,65 +0,0 @@
-from dataclasses import dataclass
-
-import numpy as np
-import mediapipe as mp
-
-
-@dataclass
-class MediaPipeSkeletonDetection:
- model_complexity: int = 1 # can be 0,1, or 2 - higher numbers are more accurate but heavier computationally
- min_detection_confidence: float = .5
- min_tracking_confidence: float = .5
-
- def __post_init__(self):
- self._mp_drawing = mp.solutions.drawing_utils
- self._mp_drawing_styles = mp.solutions.drawing_styles
- self._mp_holistic = mp.solutions.holistic
-
- self._body_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=.2)
- self._hand_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=.2)
- self._face_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=.2)
-
- self._holistic_tracker = self._mp_holistic.Holistic(model_complexity=self.model_complexity,
- min_detection_confidence=self.min_detection_confidence,
- min_tracking_confidence=self.min_tracking_confidence)
-
- def detect_skeleton_in_image(self, raw_image, annotate_image=True):
- mediapipe_results = self._holistic_tracker.process(raw_image) # <-this is where the magic happens
- if annotate_image:
- return self._annotate_image(raw_image, mediapipe_results)
-
-
- def _annotate_image(self, image, mediapipe_results):
- self._mp_drawing.draw_landmarks(image=image,
- landmark_list=mediapipe_results.face_landmarks,
- connections=self._mp_holistic.FACEMESH_CONTOURS,
- landmark_drawing_spec=None,
- connection_drawing_spec=self._mp_drawing_styles
- .get_default_face_mesh_contours_style())
- self._mp_drawing.draw_landmarks(image=image,
- landmark_list=mediapipe_results.face_landmarks,
- connections=self._mp_holistic.FACEMESH_TESSELATION,
- landmark_drawing_spec=None,
- connection_drawing_spec=self._mp_drawing_styles
- .get_default_face_mesh_tesselation_style())
- self._mp_drawing.draw_landmarks(image=image,
- landmark_list=mediapipe_results.pose_landmarks,
- connections=self._mp_holistic.POSE_CONNECTIONS,
- landmark_drawing_spec=self._mp_drawing_styles
- .get_default_pose_landmarks_style())
- self._mp_drawing.draw_landmarks(
- image=image,
- landmark_list=mediapipe_results.left_hand_landmarks,
- connections=self._mp_holistic.HAND_CONNECTIONS,
- landmark_drawing_spec=None,
- connection_drawing_spec=self._mp_drawing_styles
- .get_default_hand_connections_style())
-
- self._mp_drawing.draw_landmarks(
- image=image,
- landmark_list=mediapipe_results.right_hand_landmarks,
- connections=self._mp_holistic.HAND_CONNECTIONS,
- landmark_drawing_spec=None,
- connection_drawing_spec=self._mp_drawing_styles
- .get_default_hand_connections_style())
- return image
diff --git a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
new file mode 100644
index 00000000..2f9a7d69
--- /dev/null
+++ b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
@@ -0,0 +1,125 @@
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List
+
+import cv2
+import numpy as np
+import mediapipe as mp
+from rich.progress import Progress
+
+from jon_scratch.opencv_camera import TweakedModel
+from src.cameras.capture.dataclasses.frame_payload import FramePayload
+from src.cameras.persistence.video_writer.video_recorder import VideoRecorder
+from src.config.home_dir import get_session_path, get_synchronized_videos_path
+logger = logging.getLogger(__name__)
+
+
+class MediaPipeSkeletonDetector:
+ def __init__(self, session_id: str = None):
+ self._session_id = session_id
+ self.model_complexity = 2 # can be 0,1, or 2 - higher numbers are more accurate but heavier computationally
+ self.min_detection_confidence = .5
+ self.min_tracking_confidence = .5
+
+ self._mediapipe_payload_list=[]
+
+ self._mp_drawing = mp.solutions.drawing_utils
+ self._mp_drawing_styles = mp.solutions.drawing_styles
+ self._mp_holistic = mp.solutions.holistic
+
+ self._body_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
+ self._hand_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
+ self._face_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
+
+ self._holistic_tracker = self._mp_holistic.Holistic(model_complexity=self.model_complexity,
+ min_detection_confidence=self.min_detection_confidence,
+ min_tracking_confidence=self.min_tracking_confidence)
+
+ def detect_skeleton_in_image(self, raw_image, annotate_image=True):
+ mediapipe_results = self._holistic_tracker.process(raw_image) # <-this is where the magic happens
+ annotated_image = None
+ if annotate_image:
+ annotated_image = self._annotate_image(raw_image, mediapipe_results)
+
+ return MediapipeSkeletonPayload(mediapipe_results=mediapipe_results,
+ annotated_image=annotated_image)
+
+ def _annotate_image(self, image, mediapipe_results):
+ self._mp_drawing.draw_landmarks(image=image,
+ landmark_list=mediapipe_results.face_landmarks,
+ connections=self._mp_holistic.FACEMESH_CONTOURS,
+ landmark_drawing_spec=None,
+ connection_drawing_spec=self._mp_drawing_styles
+ .get_default_face_mesh_contours_style())
+ self._mp_drawing.draw_landmarks(image=image,
+ landmark_list=mediapipe_results.face_landmarks,
+ connections=self._mp_holistic.FACEMESH_TESSELATION,
+ landmark_drawing_spec=None,
+ connection_drawing_spec=self._mp_drawing_styles
+ .get_default_face_mesh_tesselation_style())
+ self._mp_drawing.draw_landmarks(image=image,
+ landmark_list=mediapipe_results.pose_landmarks,
+ connections=self._mp_holistic.POSE_CONNECTIONS,
+ landmark_drawing_spec=self._mp_drawing_styles
+ .get_default_pose_landmarks_style())
+ self._mp_drawing.draw_landmarks(
+ image=image,
+ landmark_list=mediapipe_results.left_hand_landmarks,
+ connections=self._mp_holistic.HAND_CONNECTIONS,
+ landmark_drawing_spec=None,
+ connection_drawing_spec=self._mp_drawing_styles
+ .get_default_hand_connections_style())
+
+ self._mp_drawing.draw_landmarks(
+ image=image,
+ landmark_list=mediapipe_results.right_hand_landmarks,
+ connections=self._mp_holistic.HAND_CONNECTIONS,
+ landmark_drawing_spec=None,
+ connection_drawing_spec=self._mp_drawing_styles
+ .get_default_hand_connections_style())
+ return image
+
+ def process_session_folder(self,
+ save_annotated_videos:bool=True):
+ synchronized_videos_path = Path(get_synchronized_videos_path(self._session_id))
+
+ for video_number, this_synchronized_video_file_path in enumerate(synchronized_videos_path.glob('*.mp4')):
+ this_video_capture_object = cv2.VideoCapture(str(this_synchronized_video_file_path))
+ logger.info(f"Running `mediapipe` skeleton detection on video: {str(this_synchronized_video_file_path)}")
+ this_video_mediapipe_payload_list = []
+
+ success=True
+ while success:
+ success, image = this_video_capture_object.read()
+ if image is None:
+ logger.error(f"Failed to load an image from: {str(this_synchronized_video_file_path)}")
+ raise Exception
+ mediapipe_skeleton_payload = self.detect_skeleton_in_image(image, annotate_image=True)
+ this_video_mediapipe_payload_list.append(mediapipe_skeleton_payload)
+
+ if save_annotated_videos:
+ self.save_annotated_videos(this_video_mediapipe_payload_list, this_video_capture_object, this_synchronized_video_file_path.stem)
+
+ def save_annotated_videos(self, mediapipe_payload_list:List, video_capture_object:cv2.VideoCapture, video_file_name:str):
+
+ this_video_name = video_file_name + "_mediapipe.mp4"
+ image_width = video_capture_object.get(cv2.CAP_PROP_FRAME_WIDTH)
+ image_height= video_capture_object.get(cv2.CAP_PROP_FRAME_HEIGHT)
+
+ video_recorder = VideoRecorder(this_video_name,
+ image_width,
+ image_height,
+ self._session_id,
+ mediapipe_annotated_video_bool=True)
+
+ image_list = [this_mediapipe_payload.annotated_image for this_mediapipe_payload in mediapipe_payload_list]
+ logger.info(f'Saving mediapipe annotated video: {video_recorder.path_to_save_video_file}')
+ video_recorder.save_image_list_to_disk(image_list, frame_rate=30)
+
+
+
+
+class MediapipeSkeletonPayload(TweakedModel):
+ mediapipe_results_list = List
+ annotated_image = FramePayload
diff --git a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py
index b432b6d1..3c99868a 100644
--- a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py
+++ b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py
@@ -22,14 +22,19 @@ logger = logging.getLogger(__name__)
class CalibrationPipelineOrchestrator:
- def __init__(self, session_id:str=None):
+ def __init__(self,
+ session_id: str = None,
+ expected_framerate: Union[int, float] = None,
+ ):
if session_id is not None:
self._session_id = session_id
self._calibration_start_time_unix_ns = time.time_ns()
self._charuco_board_detector = CharucoBoardDetector()
self._visualizer_gui = QTVisualizerAndGui(source='calibration')
- self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id)
+ self._expected_framerate = expected_framerate
+ self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id,
+ expected_framerate=self._expected_framerate)
@property
def session_id(self):
@@ -48,25 +53,27 @@ class CalibrationPipelineOrchestrator:
):
"""open all cameras and start recording, detect charuco boards until the user closes the windows, then send recorded videos to `aniposelib` for processing
produces `[session_id]_calibration.toml` file that can be used to estiamte 3d trajectorires from synchronized images"""
- with self._open_cv_camera_manager.start_capture_session_all_cams(calibration_videos=True) as connected_cameras_dict:
+ with self._open_cv_camera_manager.start_capture_session_all_cams(
+ calibration_videos=True,
+ ) as connected_cameras_dict:
+ timestamp_manager = self._open_cv_camera_manager.timestamp_manager
try:
if show_visualizer_gui:
self._visualizer_gui.setup_and_launch(self._open_cv_camera_manager.available_webcam_ids)
- timestamp_manager = TimestampManager(self._open_cv_camera_manager.available_webcam_ids,
- self._calibration_start_time_unix_ns)
-
should_continue = True
while should_continue:
- timestamp_manager.increment_main_loop_timestamp_logger(time.perf_counter_ns())
+ timestamp_manager.log_new_timestamp_for_main_loop_ns(time.perf_counter_ns())
- for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():
+ if not self._open_cv_camera_manager.new_multi_frame_ready():
+ continue
- if not this_open_cv_camera.new_frame_ready:
- continue
+ this_multi_frame_payload = self._open_cv_camera_manager.latest_multi_frame
+
+ for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():
- this_cam_latest_frame = this_open_cv_camera.latest_frame
+ this_cam_latest_frame = this_multi_frame_payload.frames_dict[this_webcam_id]
if this_cam_latest_frame is None:
continue
@@ -74,17 +81,17 @@ class CalibrationPipelineOrchestrator:
# log timestamp
this_cam_this_frame_timestamp_ns = this_cam_latest_frame.timestamp
this_cam_this_frame_number = this_cam_latest_frame.frame_number
- timestamp_manager.increment_frame_processed_for_webcam(this_webcam_id,
- this_cam_this_frame_timestamp_ns,
- this_cam_this_frame_number)
+ timestamp_manager.log_new_timestamp_for_webcam_ns(this_webcam_id,
+ this_cam_this_frame_timestamp_ns,
+ this_cam_this_frame_number)
# save frame to video file
if save_video_in_frame_loop:
# save this frame straight to the video file (no risk of memory overflow, but can't handle higher numbers of cameras)
- this_open_cv_camera.video_recorder.save_frame_to_video_file(this_cam_latest_frame)
+ this_open_cv_camera.video_recorder.save_frame_payload_to_video_file(this_cam_latest_frame)
else:
# can handle large numbers of cameras, but will eventually fill up RAM and cause a crash
- this_open_cv_camera.video_recorder.append_frame_to_list(this_cam_latest_frame)
+ this_open_cv_camera.video_recorder.append_frame_payload_to_list(this_cam_latest_frame)
# detect charuco board
this_charuco_frame_payload = self._charuco_board_detector.detect_charuco_board(
@@ -119,7 +126,7 @@ class CalibrationPipelineOrchestrator:
cv2.destroyAllWindows()
for this_open_cv_camera in connected_cameras_dict.values():
if not save_video_in_frame_loop:
- this_open_cv_camera.video_recorder.save_frame_list_to_disk()
+ this_open_cv_camera.video_recorder.save_frame_payload_list_to_disk()
this_open_cv_camera.video_recorder.close()
if show_visualizer_gui:
@@ -127,7 +134,7 @@ class CalibrationPipelineOrchestrator:
def run_anipose_camera_calibration(self,
charuco_square_size: Union[int, float] = 1,
- pin_camera_0_to_origin:bool = False,
+ pin_camera_0_to_origin: bool = False,
):
anipose_camera_calibrator = AniposeCameraCalibrator(self.session_id,
self._charuco_board_detector.charuco_board_data_class_object,
diff --git a/src/pipelines/session_pipeline/data_classes/multi_frame_payload.py b/src/pipelines/session_pipeline/data_classes/multi_frame_payload.py
new file mode 100644
index 00000000..ac741eae
--- /dev/null
+++ b/src/pipelines/session_pipeline/data_classes/multi_frame_payload.py
@@ -0,0 +1,13 @@
+from typing import NamedTuple, Union, Dict, List
+
+import numpy as np
+
+from src.cameras.capture.dataclasses.frame_payload import FramePayload
+
+
+class MultiFramePayload(NamedTuple):
+ frames_dict: Dict[str, FramePayload]
+ multi_frame_number: int
+ intra_frame_interval: Union[int, float]
+ each_frame_timestamp: List[Union[int, float]]
+ multi_frame_timestamp: Union[int, float]
diff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
index 89e61da0..6aa3dcf2 100644
--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
@@ -10,14 +10,15 @@ import numpy as np
from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager
from src.cameras.persistence.video_writer.save_options_dataclass import SaveOptions
from src.config.data_paths import freemocap_data_path
-from src.config.home_dir import create_session_id
+from src.config.home_dir import create_session_id, get_session_path
from src.core_processor.camera_calibration.camera_calibrator import CameraCalibrator
from src.core_processor.fps.timestamp_manager import TimestampManager
-from src.core_processor.mediapipe_skeleton_detection.mediapipe_skeleton_detection import MediaPipeSkeletonDetection
+from src.core_processor.mediapipe_skeleton_detector.mediapipe_skeleton_detector import MediaPipeSkeletonDetector
from src.core_processor.show_cam_window import show_cam_window
from src.core_processor.utils.image_fps_writer import write_fps_to_image
from src.debug_plot_makers.simple_plot_3d_points import simple_plot_3d
from src.pipelines.calibration_pipeline.calibration_pipeline_orchestrator import CalibrationPipelineOrchestrator
+from src.pipelines.calibration_pipeline.charuco_board_detection.charuco_board_detector import CharucoBoardDetector
from src.pipelines.session_pipeline.data_classes.data_3d_single_frame_payload import Data3dSingleFramePayload
from src.qt_visualizer_and_gui.qt_visualizer_and_gui import QTVisualizerAndGui
@@ -31,30 +32,37 @@ def get_canonical_time_str():
class SessionPipelineOrchestrator:
- def __init__(self, session_id: str = None):
+ def __init__(self, session_id: str = None, expected_framerate: Union[int, float] = None):
if session_id is not None:
self._session_id = session_id
else:
self._session_id = create_session_id()
- self._session_start_time_unix_ns = time.time_ns()
- self._visualizer_gui = QTVisualizerAndGui()
- self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id)
- self._camera_calibrator = CameraCalibrator()
- self._charuco_board_detector = self._camera_calibrator.charuco_board_detector
- self._mediapipe_skeleton_detector = MediaPipeSkeletonDetection(model_complexity=2)
+ self._expected_framerate = expected_framerate
- self._multi_frame_timestamp_intervals_sec = []
+ self._visualizer_gui = QTVisualizerAndGui()
+ self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id,
+ expected_framerate=self.expected_framerate)
+ self._charuco_board_detector = CharucoBoardDetector()
+ self._mediapipe_skeleton_detector = MediaPipeSkeletonDetector(self._session_id)
@property
def session_id(self):
return self._session_id
+ @property
+ def expected_framerate(self):
+ return self._expected_framerate
+
@property
def session_folder_path(self):
return freemocap_data_path / self._session_id
+ @property
+ def mediapipe_skeleton_detector(self):
+ return self._mediapipe_skeleton_detector
+
async def process_by_cam_id(self, webcam_id: str, cb):
with self._open_cv_camera_manager.start_capture_session_single_cam(
webcam_id=webcam_id,
@@ -73,7 +81,7 @@ class SessionPipelineOrchestrator:
charuco_frame_payload = self._camera_calibrator.calibrate(cv_cam)
if cb and charuco_frame_payload.annotated_image is not None:
- writer.append_frame_to_list(charuco_frame_payload.raw_frame_payload)
+ writer.append_frame_payload_to_list(charuco_frame_payload.raw_frame_payload)
await cb(charuco_frame_payload.annotated_image)
except Exception as e:
logger.error("Printing traceback")
@@ -91,7 +99,7 @@ class SessionPipelineOrchestrator:
frame_width=cv_cam.image_width(),
frame_height=cv_cam.image_height(),
)
- writer.save_frame_list_to_disk(options)
+ writer.save_frame_payload_list_to_disk(options)
def calibrate_camera_capture_volume(self, use_most_recent_calibration: bool = False,
load_calibration_from_session_id: str = None,
@@ -104,7 +112,8 @@ class SessionPipelineOrchestrator:
elif load_calibration_from_session_id is not None:
calibration_orchestrator = CalibrationPipelineOrchestrator(load_calibration_from_session_id)
else: # create new calibration
- calibration_orchestrator = CalibrationPipelineOrchestrator(self.session_id)
+ calibration_orchestrator = CalibrationPipelineOrchestrator(self.session_id,
+ expected_framerate=self.expected_framerate)
calibration_orchestrator.record_videos(show_visualizer_gui=False,
save_video_in_frame_loop=False,
show_camera_views_in_windows=True,
@@ -114,56 +123,37 @@ class SessionPipelineOrchestrator:
charuco_square_size=charuco_square_size,
pin_camera_0_to_origin=pin_camera_0_to_origin)
- def run_big_old_bongo_frame_loop(
+ def run_main_frame_loop(
self,
- show_camera_views_in_windows=True,
show_visualizer_gui=True,
- detect_mediapipe=True,
- detect_charuco=True,
- reconstruct_3d=True,
- save_video=True,
+ show_camera_views_in_windows=False,
+ detect_mediapipe=False,
+ detect_charuco=False,
):
- self._multi_frames_processed = 0
with self._open_cv_camera_manager.start_capture_session_all_cams() as connected_cameras_dict:
- self._number_of_cameras = len(connected_cameras_dict)
- if detect_charuco:
- incoming_charuco_frame_data_per_camera_dict = {}
- for this_webcam_id in connected_cameras_dict.keys():
- incoming_charuco_frame_data_per_camera_dict[this_webcam_id] = []
+
+ timestamp_manager = self._open_cv_camera_manager.timestamp_manager
try:
if show_visualizer_gui:
self._visualizer_gui.setup_and_launch(self._open_cv_camera_manager.available_webcam_ids)
- if detect_charuco:
- self._visualizer_gui.initialize_charuco_dottos(
- self._charuco_board_detector.number_of_charuco_corners)
-
- timestamp_manager = TimestampManager(self._open_cv_camera_manager.available_webcam_ids,
- self._session_start_time_unix_ns)
should_continue = True
while should_continue: # BIG FRAME LOOP STARTS HERE
- timestamp_manager.increment_main_loop_timestamp_logger(time.perf_counter_ns())
+ timestamp_manager.log_new_timestamp_for_main_loop_ns(time.perf_counter_ns())
- # reconstruct 3d data, if there's enough new data to so
- if reconstruct_3d:
- if detect_charuco:
- charuco_frame_payload = self._reconstruct_3d_charuco(
- incoming_charuco_frame_data_per_camera_dict)
+ if not self._open_cv_camera_manager.new_multi_frame_ready():
+ continue
+ this_multi_frame_payload = self._open_cv_camera_manager.latest_multi_frame
- if charuco_frame_payload is not None and False:
- simple_plot_3d(charuco_frame_payload.data3d_trackedPointNum_xyz)
+ if this_multi_frame_payload is None:
+ continue
- if show_visualizer_gui and charuco_frame_payload is not None:
- self._visualizer_gui.update_charuco_3d_dottos(charuco_frame_payload)
for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():
- if not this_open_cv_camera.new_frame_ready:
- continue
-
- this_cam_latest_frame = this_open_cv_camera.latest_frame
+ this_cam_latest_frame = this_multi_frame_payload.frames_dict[this_webcam_id]
if this_cam_latest_frame is None:
continue
@@ -171,11 +161,18 @@ class SessionPipelineOrchestrator:
image_to_display = this_cam_latest_frame.image.copy()
this_cam_this_frame_timestamp_ns = this_cam_latest_frame.timestamp
- timestamp_manager.increment_frame_processed_for_webcam(this_webcam_id,
- this_cam_this_frame_timestamp_ns)
+ # log timestamp
+ this_cam_this_frame_timestamp_ns = this_cam_latest_frame.timestamp
+ this_cam_this_frame_number = this_cam_latest_frame.frame_number
+ timestamp_manager.log_new_timestamp_for_webcam_ns(this_webcam_id,
+ this_cam_this_frame_timestamp_ns,
+ this_cam_this_frame_number)
+
+ # # save this frame straight to the video file (no risk of memory overflow, but can't handle higher numbers of cameras)
+ # this_open_cv_camera.video_recorder.save_frame_to_video_file(this_cam_latest_frame)
- if save_video:
- this_open_cv_camera.video_recorder.append_frame_to_list(this_cam_latest_frame)
+ # can handle large numbers of cameras, but will eventually fill up RAM and cause a crash
+ this_open_cv_camera.video_recorder.append_frame_payload_to_list(this_cam_latest_frame)
if detect_mediapipe:
image_to_display = self._mediapipe_skeleton_detector.detect_skeleton_in_image(
@@ -184,9 +181,6 @@ class SessionPipelineOrchestrator:
if detect_charuco:
charuco_frame_payload = self._charuco_board_detector.detect_charuco_board(
this_cam_latest_frame)
- incoming_charuco_frame_data_per_camera_dict[this_webcam_id].append(
- charuco_frame_payload)
-
image_to_display = charuco_frame_payload.annotated_image
if show_camera_views_in_windows:
@@ -202,24 +196,28 @@ class SessionPipelineOrchestrator:
self._visualizer_gui.update_camera_view_image(this_webcam_id, image_to_display)
# self._visualizer_gui.update_timestamp_plots(timestamp_manager)
+ if self._visualizer_gui.shut_it_down:
+ logger.info("GUI closed.")
+ should_continue = False
+
# exit loop when user presses ESC key
exit_key = cv2.waitKey(1)
if exit_key == 27:
logger.info("ESC has been pressed.")
should_continue = False
-
+
except:
logger.error("Printing traceback")
traceback.print_exc()
finally:
for this_open_cv_camera in connected_cameras_dict.values():
- if save_video:
- this_open_cv_camera.video_recorder.save_frame_list_to_disk(
- self.session_folder_path / 'synchronized_videos')
- logger.info(f"Destroy window {this_open_cv_camera.webcam_id_as_str}")
- cv2.destroyWindow(this_open_cv_camera.webcam_id_as_str)
- cv2.waitKey(1)
+
+ this_open_cv_camera.video_recorder.save_frame_payload_list_to_disk()
+
+ if show_camera_views_in_windows:
+ logger.info(f"Destroy window {this_open_cv_camera.webcam_id_as_str}")
+ cv2.destroyWindow(this_open_cv_camera.webcam_id_as_str)
if show_visualizer_gui:
self._visualizer_gui.close()
@@ -251,23 +249,13 @@ class SessionPipelineOrchestrator:
this_multi_frame_charuco_data.append(this_cam_this_frame)
is_there_any_charuco_data.append(this_cam_this_frame.charuco_view_data.some_charuco_corners_found)
- timestamp_min = np.min(this_multi_frame_timestamps)
- timestamp_max = np.max(this_multi_frame_timestamps)
- this_multiframe_timestamp_interval_sec = (timestamp_max - timestamp_min) / 1e9
- #TODO - should bring the `TimeStampManager` into this?
- self._multi_frame_timestamp_intervals_sec.append(this_multiframe_timestamp_interval_sec)
-
- print(f"TODO:sync with timestamps, not frame index - multi_frame#: {self._multi_frames_processed - 1},"
- f" each_cam_#_of_frames: {each_cam_number_of_frames}, "
- f" intra_multi_frame_timestamp_range: {this_multiframe_timestamp_interval_sec:.3f}", )
-
min_cameras_to_reconstruct = 2
if sum(is_there_any_charuco_data) >= min_cameras_to_reconstruct: # if at least 2 cameras have any data, then we can try to make some 3d dottos
charuco2d_data_per_cam_dict = self.format_charuco2d_data(this_multi_frame_charuco_data)
charuco_3d_data_payload = self._triangulate_2d_data(charuco2d_data_per_cam_dict,
self._charuco_board_detector.number_of_charuco_corners)
- x = charuco_3d_data_payload.data3d_trackedPointNum_xyz[:,0]
+ x = charuco_3d_data_payload.data3d_trackedPointNum_xyz[:, 0]
return charuco_3d_data_payload
def format_charuco2d_data(self, this_multi_frame_charuco_data) -> Dict:
@@ -298,8 +286,8 @@ class SessionPipelineOrchestrator:
data2d_trackedPointNum_xyz_camNum = np.dstack(
each_cam2d_data_list) # stack lists (in depth) to make one numpy array with dimensions [number_of_tracked_points, XYZ, number_of_cameras]
data2d_camNum_trackedPointNum_xy = data2d_trackedPointNum_xyz_camNum.reshape(self._number_of_cameras,
- number_of_tracked_points,
- 2) # reshape to fit into anipose (new dimensions [number_of_cameras, number_of_tracked_points, XYZ]
+ number_of_tracked_points,
+ 2) # reshape to fit into anipose (new dimensions [number_of_cameras, number_of_tracked_points, XYZ]
# THIS IS WHERE THE MAGIC HAPPENS - 2d data from calibrated, synchronized cameras has now become a 3d estimate. Hurray! :`D
data3d_trackedPointNum_xyz = self._anipose_camera_calibration_object.triangulate(
@@ -323,18 +311,19 @@ if __name__ == "__main__":
print('running `session_pipeline_orchestrator` as `__main__')
length_of_one_edge_of_a_black_square_on_the_charuco_board_in_mm = 39
+ expected_framerate = 30
+
+ calibrate_cameras = False
+
+ this_session_orchestrator = SessionPipelineOrchestrator(expected_framerate=expected_framerate)
- this_session_orchestrator = SessionPipelineOrchestrator()
- # this_session_orchestrator.calibrate_camera_capture_volume(use_most_recent_calibration=True)
+ if calibrate_cameras:
+ this_session_orchestrator.calibrate_camera_capture_volume(
+ charuco_square_size=length_of_one_edge_of_a_black_square_on_the_charuco_board_in_mm,
+ pin_camera_0_to_origin=True)
- this_session_orchestrator.calibrate_camera_capture_volume(
- charuco_square_size=length_of_one_edge_of_a_black_square_on_the_charuco_board_in_mm,
- pin_camera_0_to_origin=True)
+ this_session_orchestrator.run_main_frame_loop(show_camera_views_in_windows=False,
+ show_visualizer_gui=True,
+ )
- this_session_orchestrator.run_big_old_bongo_frame_loop(show_camera_views_in_windows=False,
- save_video=False,
- show_visualizer_gui=True,
- detect_mediapipe=False,
- detect_charuco=True,
- reconstruct_3d=True,
- )
+ this_session_orchestrator.mediapipe_skeleton_detector.process_session_folder(save_annotated_videos=True)
diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
index 852aefa3..3ecd64d6 100644
--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
@@ -19,9 +19,8 @@ logger = logging.getLogger(__name__)
class QTVisualizerAndGui:
def __init__(self, source:str=None):
# https://pyqtgraph.readthedocs.io/en/latest/config_options.html
- self._camera0_id = '0'
+ self.pyqtgraph_app = pg.mkQApp('freemocap! :D')
pg.setConfigOptions(imageAxisOrder='row-major')
- self.pyqtgraph_app = pg.mkQApp("FreeMoCap! :O ")
self._dict_of_camera_image_item_widgets = {}
self._dict_of_simple_timestamp_line_plots = {}
@@ -33,14 +32,24 @@ class QTVisualizerAndGui:
self._webcam_ids_list = []
self._is_paused = False
+ self._shut_it_down = False
+
+ @property
+ def shut_it_down(self):
+ return self._shut_it_down
@property
def pause_button_pressed(self):
return self._is_paused
+ def _close_button_pressed(self):
+ self.close()
+ self._shut_it_down = True
+
def close(self):
self._main_window_widget.close()
+
def setup_and_launch(self, webcam_ids_list):
logger.info('setting up QT Visualizer and GUI')
self._number_of_cameras = len(webcam_ids_list)
@@ -79,16 +88,16 @@ class QTVisualizerAndGui:
def _update_simple_timestamp_plot(self, timestamp_manager: TimestampManager):
for webcam_id in self._webcam_ids_list:
plot_item = self._dict_of_simple_timestamp_line_plots[webcam_id]
- plot_item.setData(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps)
+ plot_item.setData(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps_unix_ns)
def _update_timestamp_difference_plot(self, timestamp_manager: TimestampManager):
for webcam_id in self._webcam_ids_list:
plot_item = self._dict_of_simple_timestamp_line_plots[webcam_id]
- plot_item.setData(np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps))
+ plot_item.setData(np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps_unix_ns))
def _update_timestamp_difference_histogram(self, timestamp_manager: TimestampManager):
for webcam_id in self._webcam_ids_list:
- this_cam_timestamp_diffs = np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps)
+ this_cam_timestamp_diffs = np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps_unix_ns)
num_samples = this_cam_timestamp_diffs.shape[0]
counts, bin_edges = np.histogram(this_cam_timestamp_diffs,
bins=np.linspace(0, 100, 100))
@@ -131,15 +140,20 @@ class QTVisualizerAndGui:
self._play_button.clicked.connect(self._play)
control_panel_layout_widget.addWidget(self._play_button, row=2, col=0)
- self._reset_calibration_button = QtWidgets.QPushButton('Reset Calibration')
- self._reset_calibration_button.setEnabled(False)
- self._reset_calibration_button.clicked.connect(self._reset_calibration)
- control_panel_layout_widget.addWidget(self._reset_calibration_button, row=3, col=0)
-
- self._record_button = QtWidgets.QPushButton('Record')
- self._record_button.setEnabled(False)
- self._record_button.clicked.connect(self._record)
- control_panel_layout_widget.addWidget(self._record_button, row=3, col=0)
+ # self._reset_calibration_button = QtWidgets.QPushButton('Reset Calibration')
+ # self._reset_calibration_button.setEnabled(False)
+ # self._reset_calibration_button.clicked.connect(self._reset_calibration)
+ # control_panel_layout_widget.addWidget(self._reset_calibration_button, row=3, col=0)
+ #
+ # self._record_button = QtWidgets.QPushButton('Record')
+ # self._record_button.setEnabled(False)
+ # self._record_button.clicked.connect(self._record)
+ # control_panel_layout_widget.addWidget(self._record_button, row=3, col=0)
+
+ self._close_button = QtWidgets.QPushButton('Close All')
+ self._close_button.setEnabled(True)
+ self._close_button.clicked.connect(self._close_button_pressed)
+ control_panel_layout_widget.addWidget(self._close_button, row=4, col=0)
self._main_dock_area.addDock(self._control_panel_dock, position='left')
diff --git a/src/tests/video_writer/test_video_writer.py b/src/tests/video_writer/test_video_writer.py
index f954ed71..97ba6fc3 100644
--- a/src/tests/video_writer/test_video_writer.py
+++ b/src/tests/video_writer/test_video_writer.py
@@ -23,7 +23,7 @@ class VideoWriterTestCase(TestCase):
image=np.random.randint(0, 4, (720, 1280, 3)), timestamp=time.time_ns()
)
vw = VideoRecorder()
- vw.append_frame_to_list(example_payload)
+ vw.append_frame_payload_to_list(example_payload)
assert vw.frame_count == 1
@@ -33,7 +33,7 @@ class VideoWriterTestCase(TestCase):
image=np.random.randint(0, 4, (720, 1280, 3)), timestamp=time.time_ns()
)
vw = VideoRecorder()
- vw.append_frame_to_list(example_payload)
+ vw.append_frame_payload_to_list(example_payload)
save_options = SaveOptions(
writer_dir=file_path,
@@ -43,7 +43,7 @@ class VideoWriterTestCase(TestCase):
fps=1,
)
- vw.save_frame_list_to_disk(save_options)
+ vw.save_frame_payload_list_to_disk(save_options)
expected_path = Path().joinpath(file_path, "movie.mp4")
assert save_options.full_path == expected_path, "Paths do not match"