Free Motion Capture for Everyone πβ¨
TypeScript
gets all the way to the point where it's saving out `mediapipe2d` data :D
Jone4e27ba164ae
diff --git a/src/api/routes/session/session_router.py b/src/api/routes/session/session_router.pyindex 2da2ce49..0326828c 100644--- a/src/api/routes/session/session_router.py+++ b/src/api/routes/session/session_router.py@@ -5,7 +5,7 @@ from typing import Optionalfrom fastapi import APIRouterfrom pydantic import BaseModel-from src.config.home_dir import os_independent_home_dir, create_session_id, get_session_path+from src.config.home_dir import os_independent_home_dir, create_session_id, get_session_folder_pathlogger = logging.getLogger(__name__)@@ -24,7 +24,7 @@ class SessionResponse(BaseModel):@session_router.post("/session/create")async def create_session(session_create_model: SessionCreateModel):session_id = create_session_id(session_create_model.user_session_tag_str)- session_path = Path(get_session_path(session_id))+ session_path = Path(get_session_folder_path(session_id))logger.info(f'Creating session folder at: {str(session_path)}')session_path.mkdir(parents=True, exist_ok=False)return SessionResponse(session_id=session_id,diff --git a/src/api/services/user_config.py b/src/api/services/user_config.pyindex 3fc700c4..f0547da4 100644--- a/src/api/services/user_config.py+++ b/src/api/services/user_config.py@@ -4,7 +4,7 @@ from pathlib import Pathfrom pydantic import BaseModel-from src.config.home_dir import get_session_path+from src.config.home_dir import get_session_folder_pathlogger = logging.getLogger(__name__)@@ -26,7 +26,7 @@ class UserConfigService:webcam_config_json = webcam_config_model.json()camera_name = get_camera_name(webcam_config_model.webcam_id)json_file_name = camera_name + '_config.json'- session_path = get_session_path(session_id)+ session_path = get_session_folder_path(session_id)webcam_config_json_path = Path(session_path) / json_file_namewith open(webcam_config_json_path, 'w') as outfile:outfile.write(str(webcam_config_json))@@ -35,7 +35,7 @@ class UserConfigService:def webcam_config_by_id(self, webcam_id: str, session_id: str) -> WebcamConfigModel:if session_id is None:return WebcamConfigModel(webcam_id=webcam_id)- session_path = get_session_path(session_id)+ session_path = get_session_folder_path(session_id)camera_name = get_camera_name(webcam_id)print(f"webcam id (again): {webcam_id}")print(f"camera name: {camera_name}")diff --git a/src/cameras/persistence/video_writer/video_recorder.py b/src/cameras/persistence/video_writer/video_recorder.pyindex f887aff0..79615d0a 100644--- a/src/cameras/persistence/video_writer/video_recorder.py+++ b/src/cameras/persistence/video_writer/video_recorder.py@@ -11,8 +11,8 @@ import pandas as pdfrom src.cameras.capture.dataclasses.frame_payload import FramePayloadfrom src.cameras.persistence.video_writer.save_options_dataclass import SaveOptionsfrom src.config.data_paths import freemocap_data_path-from src.config.home_dir import get_session_path, get_synchronized_videos_path, get_calibration_videos_path, \- get_mediapipe_annotated_videos_path+from src.config.home_dir import get_session_folder_path, get_synchronized_videos_folder_path, get_calibration_videos_folder_path, \+ get_mediapipe_annotated_videos_folder_pathfrom rich.progress import Progresslogger = logging.getLogger(__name__)@@ -38,14 +38,14 @@ class VideoRecorder:# get yr paths straightself._session_id = session_id- session_path = Path(get_session_path(self._session_id))+ session_path = Path(get_session_folder_path(self._session_id))if mediapipe_annotated_video_bool:- self._video_folder_path = Path(get_mediapipe_annotated_videos_path(self._session_id))+ self._video_folder_path = Path(get_mediapipe_annotated_videos_folder_path(self._session_id))elif calibration_video_bool:- self._video_folder_path = Path(get_calibration_videos_path(self._session_id))+ self._video_folder_path = Path(get_calibration_videos_folder_path(self._session_id))else:- self._video_folder_path = Path(get_synchronized_videos_path(self._session_id))+ self._video_folder_path = Path(get_synchronized_videos_folder_path(self._session_id))self._video_folder_path.mkdir(parents=True, exist_ok=True)video_file_name = self._video_name + '.mp4'diff --git a/src/config/home_dir.py b/src/config/home_dir.pyindex fc173639..ff1e2bad 100644--- a/src/config/home_dir.py+++ b/src/config/home_dir.py@@ -4,12 +4,6 @@ from pathlib import PathBASE_FOLDER_NAME = "freemocap_data"-def os_independent_home_dir():- return str(Path.home())--def get_freemocap_data_folder_path():- return str(Path(os_independent_home_dir(), BASE_FOLDER_NAME))-def create_session_id(string_tag: str = None):session_id = 'session_' + time.strftime("%m-%d-%Y-%H_%M_%S")if string_tag is None:@@ -18,19 +12,43 @@ def create_session_id(string_tag: str = None):return session_id + '_' + string_tag-def get_session_path(session_id: str):+def os_independent_home_dir():+ return str(Path.home())+++def get_freemocap_data_folder_path():+ freemocap_data_folder_path = Path(os_independent_home_dir(), BASE_FOLDER_NAME)+ freemocap_data_folder_path.mkdir(exist_ok=True,parents=True)+ return str(freemocap_data_folder_path)+ return str(freemocap_data_folder_path)+++def get_session_folder_path(session_id: str):base_save_path = Path(get_freemocap_data_folder_path())session_path = base_save_path / session_id+ session_path.mkdir(exist_ok=True, parents=True)return str(session_path)-def get_synchronized_videos_path(session_id: str):- synchronized_videos_path = Path(get_session_path(session_id)) / 'synchronized_videos'++def get_synchronized_videos_folder_path(session_id: str):+ synchronized_videos_path = Path(get_session_folder_path(session_id)) / 'synchronized_videos'+ synchronized_videos_path.mkdir(exist_ok=True, parents=True)return str(synchronized_videos_path)-def get_calibration_videos_path(session_id: str):- calibration_videos_path = Path(get_session_path(session_id)) / 'calibration_videos'++def get_calibration_videos_folder_path(session_id: str):+ calibration_videos_path = Path(get_session_folder_path(session_id)) / 'calibration_videos'+ calibration_videos_path.mkdir(exist_ok=True, parents=True)return str(calibration_videos_path)-def get_mediapipe_annotated_videos_path(session_id: str):- mediapipe_annotated_videos_path = Path(get_session_path(session_id)) / 'mediapipe_annotated_videos'++def get_mediapipe_annotated_videos_folder_path(session_id: str):+ mediapipe_annotated_videos_path = Path(get_session_folder_path(session_id)) / 'mediapipe_annotated_videos'+ mediapipe_annotated_videos_path.mkdir(exist_ok=True, parents=True)return str(mediapipe_annotated_videos_path)+++def get_output_data_folder_path(session_id: str):+ output_data_folder_path = Path(get_session_folder_path(session_id)) / 'output_data_files'+ output_data_folder_path.mkdir(exist_ok=True, parents=True)+ return str(output_data_folder_path)diff --git a/src/core_processor/mediapipe_skeleton_detector/medaipipe_tracked_points_names_dict.py b/src/core_processor/mediapipe_skeleton_detector/medaipipe_tracked_points_names_dict.pynew file mode 100644index 00000000..fef4281b--- /dev/null+++ b/src/core_processor/mediapipe_skeleton_detector/medaipipe_tracked_points_names_dict.py@@ -0,0 +1,91 @@+mediapipe_tracked_point_names_dict = {+ "body": [+ "nose",+ "left_eye_inner",+ "left_eye",+ "left_eye_outer",+ "right_eye_inner",+ "right_eye",+ "right_eye_outer",+ "left_ear",+ "right_ear",+ "mouth_left",+ "mouth_right",+ "left_shoulder",+ "right_shoulder",+ "left_elbow",+ "right_elbow",+ "left_wrist",+ "right_wrist",+ "left_pinky",+ "right_pinky",+ "left_index",+ "right_index",+ "left_thumb",+ "right_thumb",+ "left_hip",+ "right_hip",+ "left_knee",+ "right_knee",+ "left_ankle",+ "right_ankle",+ "left_heel",+ "right_heel",+ "left_foot_index",+ "right_foot_index",+ ],+ "right_hand": [+ "right_hand_wrist",+ "right_hand_thumb_cmc",+ "right_hand_thumb_mcp",+ "right_hand_thumb_ip",+ "right_hand_thumb_tip",+ "right_hand_index_finger_mcp",+ "right_hand_index_finger_pip",+ "right_hand_index_finger_dip",+ "right_hand_index_finger_tip",+ "right_hand_middle_finger_mcp",+ "right_hand_middle_finger_pip",+ "right_hand_middle_finger_dip",+ "right_hand_middle_finger_tip",+ "right_hand_ring_finger_mcp",+ "right_hand_ring_finger_pip",+ "right_hand_ring_finger_dip",+ "right_hand_ring_finger_tip",+ "right_hand_pinky_finger_mcp",+ "right_hand_pinky_finger_pip",+ "right_hand_pinky_finger_dip",+ "right_hand_pinky_finger_tip",+ ],+ "left_hand": [+ "left_hand_wrist",+ "left_hand_thumb_cmc",+ "left_hand_thumb_mcp",+ "left_hand_thumb_ip",+ "left_hand_thumb_tip",+ "left_hand_index_finger_mcp",+ "left_hand_index_finger_pip",+ "left_hand_index_finger_dip",+ "left_hand_index_finger_tip",+ "left_hand_middle_finger_mcp",+ "left_hand_middle_finger_pip",+ "left_hand_middle_finger_dip",+ "left_hand_middle_finger_tip",+ "left_hand_ring_finger_mcp",+ "left_hand_ring_finger_pip",+ "left_hand_ring_finger_dip",+ "left_hand_ring_finger_tip",+ "left_hand_pinky_finger_mcp",+ "left_hand_pinky_finger_pip",+ "left_hand_pinky_finger_dip",+ "left_hand_pinky_finger_tip",+ ],+ "face": [+ "right_eye",+ "left_eye",+ "nose_tip",+ "mouth_center",+ "right_ear_tragion",+ "left_ear_tragion"+ ]+}diff --git a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.pyindex 2f9a7d69..8ed5ad86 100644--- a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py+++ b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py@@ -1,7 +1,7 @@import loggingfrom dataclasses import dataclassfrom pathlib import Path-from typing import List+from typing import List, Dict, Unionimport cv2import numpy as np@@ -11,10 +11,32 @@ from rich.progress import Progressfrom jon_scratch.opencv_camera import TweakedModelfrom src.cameras.capture.dataclasses.frame_payload import FramePayloadfrom src.cameras.persistence.video_writer.video_recorder import VideoRecorder-from src.config.home_dir import get_session_path, get_synchronized_videos_path+from src.config.home_dir import get_session_folder_path, get_synchronized_videos_folder_path, \+ get_output_data_folder_path+from src.core_processor.mediapipe_skeleton_detector.medaipipe_tracked_points_names_dict import \+ mediapipe_tracked_point_names_dict+logger = logging.getLogger(__name__)+@dataclass+class Mediapipe2dSingleCameraNpyArrays:+ body2d_frameNumber_trackedPointNumber_XY: np.ndarray = None+ rightHand2d_frameNumber_trackedPointNumber_XY: np.ndarray = None+ leftHand2d_frameNumber_trackedPointNumber_XY: np.ndarray = None+ face2d_frameNumber_trackedPointNumber_XY: np.ndarray = None++ body2d_frameNumber_trackedPointNumber_confidence: np.ndarray = None++ @property+ def all_data2d_nFrames_nTrackedPts_XY(self):+ """dimensions will be [number_of_frames , number_of_markers, XY]"""+ return np.hstack([self.body2d_frameNumber_trackedPointNumber_XY,+ self.rightHand2d_frameNumber_trackedPointNumber_XY,+ self.leftHand2d_frameNumber_trackedPointNumber_XY,+ self.face2d_frameNumber_trackedPointNumber_XY])++class MediaPipeSkeletonDetector:def __init__(self, session_id: str = None):self._session_id = session_id@@ -22,7 +44,7 @@ class MediaPipeSkeletonDetector:self.min_detection_confidence = .5self.min_tracking_confidence = .5- self._mediapipe_payload_list=[]+ self._mediapipe_payload_list = []self._mp_drawing = mp.solutions.drawing_utilsself._mp_drawing_styles = mp.solutions.drawing_styles@@ -35,15 +57,107 @@ class MediaPipeSkeletonDetector:self._holistic_tracker = self._mp_holistic.Holistic(model_complexity=self.model_complexity,min_detection_confidence=self.min_detection_confidence,min_tracking_confidence=self.min_tracking_confidence)+ self._mediapipe_tracked_point_names_dict = mediapipe_tracked_point_names_dictdef detect_skeleton_in_image(self, raw_image, annotate_image=True):mediapipe_results = self._holistic_tracker.process(raw_image) # <-this is where the magic happens- annotated_image = None- if annotate_image:- annotated_image = self._annotate_image(raw_image, mediapipe_results)+ return mediapipe_results++ def process_session_folder(self,+ save_annotated_videos: bool = True):+ synchronized_videos_path = Path(get_synchronized_videos_folder_path(self._session_id))++ each_video_frame_width_list = []+ each_video_frame_height_list = []++ mediapipe2d_single_camera_npy_arrays_list = []+ for video_number, this_synchronized_video_file_path in enumerate(synchronized_videos_path.glob('*.mp4')):+ this_video_capture_object = cv2.VideoCapture(str(this_synchronized_video_file_path))++ this_video_width = this_video_capture_object.get(cv2.CAP_PROP_FRAME_WIDTH)+ this_video_height = this_video_capture_object.get(cv2.CAP_PROP_FRAME_HEIGHT)++ logger.info(f"Running `mediapipe` skeleton detection on video: {str(this_synchronized_video_file_path)}")+ this_video_mediapipe_results_list = []+ this_video_annotated_images_list = []++ success, image = this_video_capture_object.read()+ if not success or image is None:+ logger.error(f"Failed to load an image from: {str(this_synchronized_video_file_path)}")+ raise Exception++ while success and image is not None:+ mediapipe_results = self.detect_skeleton_in_image(image, annotate_image=True)+ this_video_mediapipe_results_list.append(mediapipe_results)+ annotated_image = self._annotate_image(image, mediapipe_results)+ this_video_annotated_images_list.append(annotated_image)++ success, image = this_video_capture_object.read()++ if save_annotated_videos:+ self.save_annotated_videos(this_video_annotated_images_list,+ this_synchronized_video_file_path.stem,+ this_video_width,+ this_video_height,+ )++ this_camera_mediapipe_2d_single_camera_npy_arrays = self._list_of_mediapipe_results_to_npy_arrays(+ this_video_mediapipe_results_list,+ image_width=this_video_width,+ image_height=this_video_height)++ mediapipe2d_single_camera_npy_arrays_list.append(this_camera_mediapipe_2d_single_camera_npy_arrays)++ all_cameras_data2d_list = [m2d.all_data2d_nFrames_nTrackedPts_XY for m2d in+ mediapipe2d_single_camera_npy_arrays_list]++ number_of_cameras = len(all_cameras_data2d_list)+ number_of_frames = all_cameras_data2d_list[0].shape[0]+ number_of_tracked_points = all_cameras_data2d_list[0].shape[1]+ number_of_spatial_dimensions = all_cameras_data2d_list[0].shape[2] # XY, 2d data++ if not number_of_spatial_dimensions == 2:+ logger.error(f"this should be 2D data (XY pixel coordinates), but we founds {number_of_spatial_dimensions}")+ raise Exception+++ data2d_numCams_numFrames_numTrackedPts_XY = np.empty((number_of_cameras,+ number_of_frames,+ number_of_tracked_points,+ number_of_spatial_dimensions))++ for this_cam_num in range(number_of_cameras):+ data2d_numCams_numFrames_numTrackedPts_XY[this_cam_num, :, :, :] = all_cameras_data2d_list[this_cam_num]++ self._save_data_to_npy(data2d_numCams_numFrames_numTrackedPts_XY)+ return data2d_numCams_numFrames_numTrackedPts_XY+++ def _save_data_to_npy(self,data2d_numCams_numFrames_numTrackedPts_XY):+ output_data_folder = Path(get_output_data_folder_path(self._session_id))+ mediapipe_2dData_save_path = output_data_folder / "mediapipe_2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy"+ logger.info(f"saving: {mediapipe_2dData_save_path}")+ np.save(str(mediapipe_2dData_save_path), data2d_numCams_numFrames_numTrackedPts_XY)++++ def save_annotated_videos(self,+ annotated_images_list: List[np.ndarray],+ video_file_name: str,+ image_width: Union[int, float],+ image_height: Union[int, float],+ ):- return MediapipeSkeletonPayload(mediapipe_results=mediapipe_results,- annotated_image=annotated_image)+ this_video_name = video_file_name + "_mediapipe.mp4"++ video_recorder = VideoRecorder(this_video_name,+ image_width,+ image_height,+ self._session_id,+ mediapipe_annotated_video_bool=True)++ logger.info(f'Saving mediapipe annotated video: {video_recorder.path_to_save_video_file}')+ video_recorder.save_image_list_to_disk(annotated_images_list, frame_rate=30)def _annotate_image(self, image, mediapipe_results):self._mp_drawing.draw_landmarks(image=image,@@ -80,46 +194,123 @@ class MediaPipeSkeletonDetector:.get_default_hand_connections_style())return image- def process_session_folder(self,- save_annotated_videos:bool=True):- synchronized_videos_path = Path(get_synchronized_videos_path(self._session_id))+ def _list_of_mediapipe_results_to_npy_arrays(self,+ mediapipe_results_list: List,+ image_width: int = 1,+ image_height: int = 1+ ) -> Mediapipe2dSingleCameraNpyArrays:- for video_number, this_synchronized_video_file_path in enumerate(synchronized_videos_path.glob('*.mp4')):- this_video_capture_object = cv2.VideoCapture(str(this_synchronized_video_file_path))- logger.info(f"Running `mediapipe` skeleton detection on video: {str(this_synchronized_video_file_path)}")- this_video_mediapipe_payload_list = []+ body_names_list = self._mediapipe_tracked_point_names_dict["body"]+ right_hand_names_list = self._mediapipe_tracked_point_names_dict["right_hand"]+ left_hand_names_list = self._mediapipe_tracked_point_names_dict["left_hand"]+ face_names_list = self._mediapipe_tracked_point_names_dict["face"]- success=True- while success:- success, image = this_video_capture_object.read()- if image is None:- logger.error(f"Failed to load an image from: {str(this_synchronized_video_file_path)}")- raise Exception- mediapipe_skeleton_payload = self.detect_skeleton_in_image(image, annotate_image=True)- this_video_mediapipe_payload_list.append(mediapipe_skeleton_payload)+ # apparently `mediapipe_results.pose_landmarks.landmark` returns something iterable Β―\_(γ)_/Β―+ mediapipe_pose_landmark_iterator = mp.python.solutions.pose.PoseLandmark+ mediapipe_hand_landmark_iterator = mp.python.solutions.hands.HandLandmark+ # TODO - build a better iterator and list of `face_marker_names` that will only pull out the face_counters & iris edges (mp.python.solutions.face_mesh_connections.FACEMESH_CONTOURS, FACE_MESH_IRISES)- if save_annotated_videos:- self.save_annotated_videos(this_video_mediapipe_payload_list, this_video_capture_object, this_synchronized_video_file_path.stem)+ number_of_frames = len(mediapipe_results_list)- def save_annotated_videos(self, mediapipe_payload_list:List, video_capture_object:cv2.VideoCapture, video_file_name:str):+ number_of_body_trackedPoints = len(body_names_list)+ number_of_right_hand_trackedPoints = len(right_hand_names_list)+ number_of_left_hand_trackedPoints = len(left_hand_names_list)+ number_of_face_trackedPoints = mp.python.solutions.face_mesh.FACEMESH_NUM_LANDMARKS_WITH_IRISES+ number_of_spatial_dimensions = 2 # this will be 2d XY pixel data- this_video_name = video_file_name + "_mediapipe.mp4"- image_width = video_capture_object.get(cv2.CAP_PROP_FRAME_WIDTH)- image_height= video_capture_object.get(cv2.CAP_PROP_FRAME_HEIGHT)+ body2d_frameNumber_trackedPointNumber_XY = np.zeros(+ (number_of_frames, number_of_body_trackedPoints, number_of_spatial_dimensions))+ body2d_frameNumber_trackedPointNumber_XY[:] = np.nan- video_recorder = VideoRecorder(this_video_name,- image_width,- image_height,- self._session_id,- mediapipe_annotated_video_bool=True)+ body2d_frameNumber_trackedPointNumber_confidence = np.zeros((number_of_frames, number_of_body_trackedPoints))+ body2d_frameNumber_trackedPointNumber_confidence[:] = np.nan # only body markers get a 'confidence' value- image_list = [this_mediapipe_payload.annotated_image for this_mediapipe_payload in mediapipe_payload_list]- logger.info(f'Saving mediapipe annotated video: {video_recorder.path_to_save_video_file}')- video_recorder.save_image_list_to_disk(image_list, frame_rate=30)+ rightHand2d_frameNumber_trackedPointNumber_XY = np.zeros((number_of_frames,+ number_of_right_hand_trackedPoints,+ number_of_spatial_dimensions))+ rightHand2d_frameNumber_trackedPointNumber_XY[:] = np.nan++ leftHand2d_frameNumber_trackedPointNumber_XY = np.zeros((number_of_frames,+ number_of_left_hand_trackedPoints,+ number_of_spatial_dimensions))+ leftHand2d_frameNumber_trackedPointNumber_XY[:] = np.nan++ face2d_frameNumber_trackedPointNumber_XY = np.zeros((number_of_frames,+ number_of_face_trackedPoints,+ number_of_spatial_dimensions))+ face2d_frameNumber_trackedPointNumber_XY[:] = np.nan++ all_body_tracked_points_visible_on_this_frame_bool_list = []+ all_right_hand_points_visible_on_this_frame_bool_list = []+ all_left_hand_points_visible_on_this_frame_bool_list = []+ all_face_points_visible_on_this_frame_bool_list = []+ all_tracked_points_visible_on_this_frame_list = []++ for this_frame_number, this_frame_results in enumerate(mediapipe_results_list):++ # get the Body data (aka 'pose')+ if this_frame_results.pose_landmarks is not None:++ for this_landmark_data in this_frame_results.pose_landmarks.landmark:+ body2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,+ 0] = this_landmark_data.x * image_width+ body2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,+ 1] = this_landmark_data.y * image_height+ body2d_frameNumber_trackedPointNumber_confidence[this_frame_number,+ :] = this_landmark_data.visibility # mediapipe calls their 'confidence' score 'visibility'++ # get Right Hand data+ if this_frame_results.right_hand_landmarks is not None:+ for this_landmark_data in this_frame_results.right_hand_landmarks.landmark:+ rightHand2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,+ 0] = this_landmark_data.x * image_width+ rightHand2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,+ 1] = this_landmark_data.y * image_height++ # get Left Hand data+ if this_frame_results.left_hand_landmarks is not None:+ for this_landmark_data in this_frame_results.left_hand_landmarks.landmark:+ leftHand2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,+ 0] = this_landmark_data.x * image_width+ leftHand2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,+ 1] = this_landmark_data.y * image_height++ # get Face data+ if this_frame_results.face_landmarks is not None:+ for this_landmark_data in this_frame_results.face_landmarks.landmark:+ face2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,+ 0] = this_landmark_data.x * image_width+ face2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,+ 1] = this_landmark_data.y * image_height++ # check if all tracked points are visible on this frame+ all_body_visible = all(sum(+ np.isnan(body2d_frameNumber_trackedPointNumber_XY[this_frame_number, :, :])) == 0)+ all_body_tracked_points_visible_on_this_frame_bool_list.append(all_body_visible)++ all_right_hand_visible = all(sum(+ np.isnan(rightHand2d_frameNumber_trackedPointNumber_XY[this_frame_number, :, :])) == 0)+ all_right_hand_points_visible_on_this_frame_bool_list.append(all_right_hand_visible)++ all_left_hand_visible = all(sum(+ np.isnan(leftHand2d_frameNumber_trackedPointNumber_XY[this_frame_number, :, :])) == 0)+ all_left_hand_points_visible_on_this_frame_bool_list.append(all_left_hand_visible)+ all_face_visible = all(sum(+ np.isnan(face2d_frameNumber_trackedPointNumber_XY[this_frame_number, :, :])) == 0)+ all_face_points_visible_on_this_frame_bool_list.append(all_face_visible)+ all_points_visible = all([all_body_visible,+ all_right_hand_visible,+ all_left_hand_visible,+ all_face_visible],+ )+ all_tracked_points_visible_on_this_frame_list.append(all_points_visible)-class MediapipeSkeletonPayload(TweakedModel):- mediapipe_results_list = List- annotated_image = FramePayload+ return Mediapipe2dSingleCameraNpyArrays(+ body2d_frameNumber_trackedPointNumber_XY=body2d_frameNumber_trackedPointNumber_XY,+ rightHand2d_frameNumber_trackedPointNumber_XY=rightHand2d_frameNumber_trackedPointNumber_XY,+ leftHand2d_frameNumber_trackedPointNumber_XY=leftHand2d_frameNumber_trackedPointNumber_XY,+ face2d_frameNumber_trackedPointNumber_XY=face2d_frameNumber_trackedPointNumber_XY,+ body2d_frameNumber_trackedPointNumber_confidence=body2d_frameNumber_trackedPointNumber_confidence)diff --git a/src/pipelines/calibration_pipeline/anipose_camera_calibration/anipose_camera_calibration.py b/src/pipelines/calibration_pipeline/anipose_camera_calibration/anipose_camera_calibration.pyindex 8d641b4f..d482e8aa 100644--- a/src/pipelines/calibration_pipeline/anipose_camera_calibration/anipose_camera_calibration.py+++ b/src/pipelines/calibration_pipeline/anipose_camera_calibration/anipose_camera_calibration.py@@ -6,7 +6,8 @@ import numpy as npfrom cv2.cv2 import aruco_CharucoBoardfrom scipy.spatial.transform import Rotation-from src.config.home_dir import get_session_path, get_freemocap_data_folder_path+from src.config.home_dir import get_session_folder_path, get_freemocap_data_folder_path, \+ get_calibration_videos_folder_pathfrom src.pipelines.calibration_pipeline.anipose_camera_calibration import freemocap_aniposefrom aniposelib.boards import CharucoBoard as AniposeCharucoBoard@@ -34,8 +35,8 @@ class AniposeCameraCalibrator:self.initialize_anipose_objects()def get_paths_and_whatnot(self):- self.session_folder_path = Path(get_session_path(self._session_id))- calibration_videos_folder = self.session_folder_path / 'calibration_videos'+ self.session_folder_path = Path(get_session_folder_path(self._session_id))+ calibration_videos_folder = Path(get_calibration_videos_folder_path())self._list_of_video_paths = [this_video_path for this_video_path incalibration_videos_folder.glob('*.mp4'.lower())]diff --git a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.pyindex 3c99868a..f5acf0b4 100644--- a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py+++ b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py@@ -7,7 +7,7 @@ from typing import Unionimport cv2from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager-from src.config.home_dir import create_session_id, get_session_path, get_freemocap_data_folder_path+from src.config.home_dir import create_session_id, get_session_folder_path, get_freemocap_data_folder_pathfrom src.core_processor.fps.timestamp_manager import TimestampManagerfrom src.core_processor.show_cam_window import show_cam_windowfrom src.core_processor.utils.image_fps_writer import write_fps_to_image@@ -42,7 +42,7 @@ class CalibrationPipelineOrchestrator:@propertydef session_folder_path(self):- return get_session_path(self._session_id)+ return get_session_folder_path(self._session_id)def record_videos(self,@@ -153,7 +153,7 @@ if __name__ == "__main__":record_new = Trueif record_new:session_id = create_session_id('calibration')- session_path = Path(get_session_path(session_id))+ session_path = Path(get_session_folder_path(session_id))logger.info(f'Creating `calibration only` session folder at: {str(session_path)}')session_path.mkdir(parents=True, exist_ok=False)diff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.pyindex 6aa3dcf2..1ce30c3a 100644--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py@@ -10,7 +10,7 @@ import numpy as npfrom src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManagerfrom src.cameras.persistence.video_writer.save_options_dataclass import SaveOptionsfrom src.config.data_paths import freemocap_data_path-from src.config.home_dir import create_session_id, get_session_path+from src.config.home_dir import create_session_id, get_session_folder_pathfrom src.core_processor.camera_calibration.camera_calibrator import CameraCalibratorfrom src.core_processor.fps.timestamp_manager import TimestampManagerfrom src.core_processor.mediapipe_skeleton_detector.mediapipe_skeleton_detector import MediaPipeSkeletonDetector@@ -306,6 +306,10 @@ class SessionPipelineOrchestrator:data3d_trackedPointNum_xyz=data3d_trackedPointNum_xyz,data3d_trackedPointNum_reprojectionError=data3d_trackedPointNum_reprojectionError, )+ def mediapipe_track_skeletons(self):+ self._mediapipe2d_numCams_numFrames_numTrackedPoints_XY = self.mediapipe_skeleton_detector.process_session_folder(save_annotated_videos=True)+ pass+if __name__ == "__main__":print('running `session_pipeline_orchestrator` as `__main__')@@ -326,4 +330,4 @@ if __name__ == "__main__":show_visualizer_gui=True,)- this_session_orchestrator.mediapipe_skeleton_detector.process_session_folder(save_annotated_videos=True)+ this_session_orchestrator.mediapipe_track_skeletons()