Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

gets all the way to the point where it's saving out `mediapipe2d` data :D

Jone4e27ba164ae
diff --git a/src/api/routes/session/session_router.py b/src/api/routes/session/session_router.py
index 2da2ce49..0326828c 100644
--- a/src/api/routes/session/session_router.py
+++ b/src/api/routes/session/session_router.py
@@ -5,7 +5,7 @@ from typing import Optional
from fastapi import APIRouter
from pydantic import BaseModel
-from src.config.home_dir import os_independent_home_dir, create_session_id, get_session_path
+from src.config.home_dir import os_independent_home_dir, create_session_id, get_session_folder_path
logger = logging.getLogger(__name__)
@@ -24,7 +24,7 @@ class SessionResponse(BaseModel):
@session_router.post("/session/create")
async def create_session(session_create_model: SessionCreateModel):
session_id = create_session_id(session_create_model.user_session_tag_str)
- session_path = Path(get_session_path(session_id))
+ session_path = Path(get_session_folder_path(session_id))
logger.info(f'Creating session folder at: {str(session_path)}')
session_path.mkdir(parents=True, exist_ok=False)
return SessionResponse(session_id=session_id,
diff --git a/src/api/services/user_config.py b/src/api/services/user_config.py
index 3fc700c4..f0547da4 100644
--- a/src/api/services/user_config.py
+++ b/src/api/services/user_config.py
@@ -4,7 +4,7 @@ from pathlib import Path
from pydantic import BaseModel
-from src.config.home_dir import get_session_path
+from src.config.home_dir import get_session_folder_path
logger = logging.getLogger(__name__)
@@ -26,7 +26,7 @@ class UserConfigService:
webcam_config_json = webcam_config_model.json()
camera_name = get_camera_name(webcam_config_model.webcam_id)
json_file_name = camera_name + '_config.json'
- session_path = get_session_path(session_id)
+ session_path = get_session_folder_path(session_id)
webcam_config_json_path = Path(session_path) / json_file_name
with open(webcam_config_json_path, 'w') as outfile:
outfile.write(str(webcam_config_json))
@@ -35,7 +35,7 @@ class UserConfigService:
def webcam_config_by_id(self, webcam_id: str, session_id: str) -> WebcamConfigModel:
if session_id is None:
return WebcamConfigModel(webcam_id=webcam_id)
- session_path = get_session_path(session_id)
+ session_path = get_session_folder_path(session_id)
camera_name = get_camera_name(webcam_id)
print(f"webcam id (again): {webcam_id}")
print(f"camera name: {camera_name}")
diff --git a/src/cameras/persistence/video_writer/video_recorder.py b/src/cameras/persistence/video_writer/video_recorder.py
index f887aff0..79615d0a 100644
--- a/src/cameras/persistence/video_writer/video_recorder.py
+++ b/src/cameras/persistence/video_writer/video_recorder.py
@@ -11,8 +11,8 @@ import pandas as pd
from src.cameras.capture.dataclasses.frame_payload import FramePayload
from src.cameras.persistence.video_writer.save_options_dataclass import SaveOptions
from src.config.data_paths import freemocap_data_path
-from src.config.home_dir import get_session_path, get_synchronized_videos_path, get_calibration_videos_path, \
- get_mediapipe_annotated_videos_path
+from src.config.home_dir import get_session_folder_path, get_synchronized_videos_folder_path, get_calibration_videos_folder_path, \
+ get_mediapipe_annotated_videos_folder_path
from rich.progress import Progress
logger = logging.getLogger(__name__)
@@ -38,14 +38,14 @@ class VideoRecorder:
# get yr paths straight
self._session_id = session_id
- session_path = Path(get_session_path(self._session_id))
+ session_path = Path(get_session_folder_path(self._session_id))
if mediapipe_annotated_video_bool:
- self._video_folder_path = Path(get_mediapipe_annotated_videos_path(self._session_id))
+ self._video_folder_path = Path(get_mediapipe_annotated_videos_folder_path(self._session_id))
elif calibration_video_bool:
- self._video_folder_path = Path(get_calibration_videos_path(self._session_id))
+ self._video_folder_path = Path(get_calibration_videos_folder_path(self._session_id))
else:
- self._video_folder_path = Path(get_synchronized_videos_path(self._session_id))
+ self._video_folder_path = Path(get_synchronized_videos_folder_path(self._session_id))
self._video_folder_path.mkdir(parents=True, exist_ok=True)
video_file_name = self._video_name + '.mp4'
diff --git a/src/config/home_dir.py b/src/config/home_dir.py
index fc173639..ff1e2bad 100644
--- a/src/config/home_dir.py
+++ b/src/config/home_dir.py
@@ -4,12 +4,6 @@ from pathlib import Path
BASE_FOLDER_NAME = "freemocap_data"
-def os_independent_home_dir():
- return str(Path.home())
-
-def get_freemocap_data_folder_path():
- return str(Path(os_independent_home_dir(), BASE_FOLDER_NAME))
-
def create_session_id(string_tag: str = None):
session_id = 'session_' + time.strftime("%m-%d-%Y-%H_%M_%S")
if string_tag is None:
@@ -18,19 +12,43 @@ def create_session_id(string_tag: str = None):
return session_id + '_' + string_tag
-def get_session_path(session_id: str):
+def os_independent_home_dir():
+ return str(Path.home())
+
+
+def get_freemocap_data_folder_path():
+ freemocap_data_folder_path = Path(os_independent_home_dir(), BASE_FOLDER_NAME)
+ freemocap_data_folder_path.mkdir(exist_ok=True,parents=True)
+ return str(freemocap_data_folder_path)
+ return str(freemocap_data_folder_path)
+
+
+def get_session_folder_path(session_id: str):
base_save_path = Path(get_freemocap_data_folder_path())
session_path = base_save_path / session_id
+ session_path.mkdir(exist_ok=True, parents=True)
return str(session_path)
-def get_synchronized_videos_path(session_id: str):
- synchronized_videos_path = Path(get_session_path(session_id)) / 'synchronized_videos'
+
+def get_synchronized_videos_folder_path(session_id: str):
+ synchronized_videos_path = Path(get_session_folder_path(session_id)) / 'synchronized_videos'
+ synchronized_videos_path.mkdir(exist_ok=True, parents=True)
return str(synchronized_videos_path)
-def get_calibration_videos_path(session_id: str):
- calibration_videos_path = Path(get_session_path(session_id)) / 'calibration_videos'
+
+def get_calibration_videos_folder_path(session_id: str):
+ calibration_videos_path = Path(get_session_folder_path(session_id)) / 'calibration_videos'
+ calibration_videos_path.mkdir(exist_ok=True, parents=True)
return str(calibration_videos_path)
-def get_mediapipe_annotated_videos_path(session_id: str):
- mediapipe_annotated_videos_path = Path(get_session_path(session_id)) / 'mediapipe_annotated_videos'
+
+def get_mediapipe_annotated_videos_folder_path(session_id: str):
+ mediapipe_annotated_videos_path = Path(get_session_folder_path(session_id)) / 'mediapipe_annotated_videos'
+ mediapipe_annotated_videos_path.mkdir(exist_ok=True, parents=True)
return str(mediapipe_annotated_videos_path)
+
+
+def get_output_data_folder_path(session_id: str):
+ output_data_folder_path = Path(get_session_folder_path(session_id)) / 'output_data_files'
+ output_data_folder_path.mkdir(exist_ok=True, parents=True)
+ return str(output_data_folder_path)
diff --git a/src/core_processor/mediapipe_skeleton_detector/medaipipe_tracked_points_names_dict.py b/src/core_processor/mediapipe_skeleton_detector/medaipipe_tracked_points_names_dict.py
new file mode 100644
index 00000000..fef4281b
--- /dev/null
+++ b/src/core_processor/mediapipe_skeleton_detector/medaipipe_tracked_points_names_dict.py
@@ -0,0 +1,91 @@
+mediapipe_tracked_point_names_dict = {
+ "body": [
+ "nose",
+ "left_eye_inner",
+ "left_eye",
+ "left_eye_outer",
+ "right_eye_inner",
+ "right_eye",
+ "right_eye_outer",
+ "left_ear",
+ "right_ear",
+ "mouth_left",
+ "mouth_right",
+ "left_shoulder",
+ "right_shoulder",
+ "left_elbow",
+ "right_elbow",
+ "left_wrist",
+ "right_wrist",
+ "left_pinky",
+ "right_pinky",
+ "left_index",
+ "right_index",
+ "left_thumb",
+ "right_thumb",
+ "left_hip",
+ "right_hip",
+ "left_knee",
+ "right_knee",
+ "left_ankle",
+ "right_ankle",
+ "left_heel",
+ "right_heel",
+ "left_foot_index",
+ "right_foot_index",
+ ],
+ "right_hand": [
+ "right_hand_wrist",
+ "right_hand_thumb_cmc",
+ "right_hand_thumb_mcp",
+ "right_hand_thumb_ip",
+ "right_hand_thumb_tip",
+ "right_hand_index_finger_mcp",
+ "right_hand_index_finger_pip",
+ "right_hand_index_finger_dip",
+ "right_hand_index_finger_tip",
+ "right_hand_middle_finger_mcp",
+ "right_hand_middle_finger_pip",
+ "right_hand_middle_finger_dip",
+ "right_hand_middle_finger_tip",
+ "right_hand_ring_finger_mcp",
+ "right_hand_ring_finger_pip",
+ "right_hand_ring_finger_dip",
+ "right_hand_ring_finger_tip",
+ "right_hand_pinky_finger_mcp",
+ "right_hand_pinky_finger_pip",
+ "right_hand_pinky_finger_dip",
+ "right_hand_pinky_finger_tip",
+ ],
+ "left_hand": [
+ "left_hand_wrist",
+ "left_hand_thumb_cmc",
+ "left_hand_thumb_mcp",
+ "left_hand_thumb_ip",
+ "left_hand_thumb_tip",
+ "left_hand_index_finger_mcp",
+ "left_hand_index_finger_pip",
+ "left_hand_index_finger_dip",
+ "left_hand_index_finger_tip",
+ "left_hand_middle_finger_mcp",
+ "left_hand_middle_finger_pip",
+ "left_hand_middle_finger_dip",
+ "left_hand_middle_finger_tip",
+ "left_hand_ring_finger_mcp",
+ "left_hand_ring_finger_pip",
+ "left_hand_ring_finger_dip",
+ "left_hand_ring_finger_tip",
+ "left_hand_pinky_finger_mcp",
+ "left_hand_pinky_finger_pip",
+ "left_hand_pinky_finger_dip",
+ "left_hand_pinky_finger_tip",
+ ],
+ "face": [
+ "right_eye",
+ "left_eye",
+ "nose_tip",
+ "mouth_center",
+ "right_ear_tragion",
+ "left_ear_tragion"
+ ]
+}
diff --git a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
index 2f9a7d69..8ed5ad86 100644
--- a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
+++ b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
@@ -1,7 +1,7 @@
import logging
from dataclasses import dataclass
from pathlib import Path
-from typing import List
+from typing import List, Dict, Union
import cv2
import numpy as np
@@ -11,10 +11,32 @@ from rich.progress import Progress
from jon_scratch.opencv_camera import TweakedModel
from src.cameras.capture.dataclasses.frame_payload import FramePayload
from src.cameras.persistence.video_writer.video_recorder import VideoRecorder
-from src.config.home_dir import get_session_path, get_synchronized_videos_path
+from src.config.home_dir import get_session_folder_path, get_synchronized_videos_folder_path, \
+ get_output_data_folder_path
+from src.core_processor.mediapipe_skeleton_detector.medaipipe_tracked_points_names_dict import \
+ mediapipe_tracked_point_names_dict
+
logger = logging.getLogger(__name__)
+@dataclass
+class Mediapipe2dSingleCameraNpyArrays:
+ body2d_frameNumber_trackedPointNumber_XY: np.ndarray = None
+ rightHand2d_frameNumber_trackedPointNumber_XY: np.ndarray = None
+ leftHand2d_frameNumber_trackedPointNumber_XY: np.ndarray = None
+ face2d_frameNumber_trackedPointNumber_XY: np.ndarray = None
+
+ body2d_frameNumber_trackedPointNumber_confidence: np.ndarray = None
+
+ @property
+ def all_data2d_nFrames_nTrackedPts_XY(self):
+ """dimensions will be [number_of_frames , number_of_markers, XY]"""
+ return np.hstack([self.body2d_frameNumber_trackedPointNumber_XY,
+ self.rightHand2d_frameNumber_trackedPointNumber_XY,
+ self.leftHand2d_frameNumber_trackedPointNumber_XY,
+ self.face2d_frameNumber_trackedPointNumber_XY])
+
+
class MediaPipeSkeletonDetector:
def __init__(self, session_id: str = None):
self._session_id = session_id
@@ -22,7 +44,7 @@ class MediaPipeSkeletonDetector:
self.min_detection_confidence = .5
self.min_tracking_confidence = .5
- self._mediapipe_payload_list=[]
+ self._mediapipe_payload_list = []
self._mp_drawing = mp.solutions.drawing_utils
self._mp_drawing_styles = mp.solutions.drawing_styles
@@ -35,15 +57,107 @@ class MediaPipeSkeletonDetector:
self._holistic_tracker = self._mp_holistic.Holistic(model_complexity=self.model_complexity,
min_detection_confidence=self.min_detection_confidence,
min_tracking_confidence=self.min_tracking_confidence)
+ self._mediapipe_tracked_point_names_dict = mediapipe_tracked_point_names_dict
def detect_skeleton_in_image(self, raw_image, annotate_image=True):
mediapipe_results = self._holistic_tracker.process(raw_image) # <-this is where the magic happens
- annotated_image = None
- if annotate_image:
- annotated_image = self._annotate_image(raw_image, mediapipe_results)
+ return mediapipe_results
+
+ def process_session_folder(self,
+ save_annotated_videos: bool = True):
+ synchronized_videos_path = Path(get_synchronized_videos_folder_path(self._session_id))
+
+ each_video_frame_width_list = []
+ each_video_frame_height_list = []
+
+ mediapipe2d_single_camera_npy_arrays_list = []
+ for video_number, this_synchronized_video_file_path in enumerate(synchronized_videos_path.glob('*.mp4')):
+ this_video_capture_object = cv2.VideoCapture(str(this_synchronized_video_file_path))
+
+ this_video_width = this_video_capture_object.get(cv2.CAP_PROP_FRAME_WIDTH)
+ this_video_height = this_video_capture_object.get(cv2.CAP_PROP_FRAME_HEIGHT)
+
+ logger.info(f"Running `mediapipe` skeleton detection on video: {str(this_synchronized_video_file_path)}")
+ this_video_mediapipe_results_list = []
+ this_video_annotated_images_list = []
+
+ success, image = this_video_capture_object.read()
+ if not success or image is None:
+ logger.error(f"Failed to load an image from: {str(this_synchronized_video_file_path)}")
+ raise Exception
+
+ while success and image is not None:
+ mediapipe_results = self.detect_skeleton_in_image(image, annotate_image=True)
+ this_video_mediapipe_results_list.append(mediapipe_results)
+ annotated_image = self._annotate_image(image, mediapipe_results)
+ this_video_annotated_images_list.append(annotated_image)
+
+ success, image = this_video_capture_object.read()
+
+ if save_annotated_videos:
+ self.save_annotated_videos(this_video_annotated_images_list,
+ this_synchronized_video_file_path.stem,
+ this_video_width,
+ this_video_height,
+ )
+
+ this_camera_mediapipe_2d_single_camera_npy_arrays = self._list_of_mediapipe_results_to_npy_arrays(
+ this_video_mediapipe_results_list,
+ image_width=this_video_width,
+ image_height=this_video_height)
+
+ mediapipe2d_single_camera_npy_arrays_list.append(this_camera_mediapipe_2d_single_camera_npy_arrays)
+
+ all_cameras_data2d_list = [m2d.all_data2d_nFrames_nTrackedPts_XY for m2d in
+ mediapipe2d_single_camera_npy_arrays_list]
+
+ number_of_cameras = len(all_cameras_data2d_list)
+ number_of_frames = all_cameras_data2d_list[0].shape[0]
+ number_of_tracked_points = all_cameras_data2d_list[0].shape[1]
+ number_of_spatial_dimensions = all_cameras_data2d_list[0].shape[2] # XY, 2d data
+
+ if not number_of_spatial_dimensions == 2:
+ logger.error(f"this should be 2D data (XY pixel coordinates), but we founds {number_of_spatial_dimensions}")
+ raise Exception
+
+
+ data2d_numCams_numFrames_numTrackedPts_XY = np.empty((number_of_cameras,
+ number_of_frames,
+ number_of_tracked_points,
+ number_of_spatial_dimensions))
+
+ for this_cam_num in range(number_of_cameras):
+ data2d_numCams_numFrames_numTrackedPts_XY[this_cam_num, :, :, :] = all_cameras_data2d_list[this_cam_num]
+
+ self._save_data_to_npy(data2d_numCams_numFrames_numTrackedPts_XY)
+ return data2d_numCams_numFrames_numTrackedPts_XY
+
+
+ def _save_data_to_npy(self,data2d_numCams_numFrames_numTrackedPts_XY):
+ output_data_folder = Path(get_output_data_folder_path(self._session_id))
+ mediapipe_2dData_save_path = output_data_folder / "mediapipe_2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy"
+ logger.info(f"saving: {mediapipe_2dData_save_path}")
+ np.save(str(mediapipe_2dData_save_path), data2d_numCams_numFrames_numTrackedPts_XY)
+
+
+
+ def save_annotated_videos(self,
+ annotated_images_list: List[np.ndarray],
+ video_file_name: str,
+ image_width: Union[int, float],
+ image_height: Union[int, float],
+ ):
- return MediapipeSkeletonPayload(mediapipe_results=mediapipe_results,
- annotated_image=annotated_image)
+ this_video_name = video_file_name + "_mediapipe.mp4"
+
+ video_recorder = VideoRecorder(this_video_name,
+ image_width,
+ image_height,
+ self._session_id,
+ mediapipe_annotated_video_bool=True)
+
+ logger.info(f'Saving mediapipe annotated video: {video_recorder.path_to_save_video_file}')
+ video_recorder.save_image_list_to_disk(annotated_images_list, frame_rate=30)
def _annotate_image(self, image, mediapipe_results):
self._mp_drawing.draw_landmarks(image=image,
@@ -80,46 +194,123 @@ class MediaPipeSkeletonDetector:
.get_default_hand_connections_style())
return image
- def process_session_folder(self,
- save_annotated_videos:bool=True):
- synchronized_videos_path = Path(get_synchronized_videos_path(self._session_id))
+ def _list_of_mediapipe_results_to_npy_arrays(self,
+ mediapipe_results_list: List,
+ image_width: int = 1,
+ image_height: int = 1
+ ) -> Mediapipe2dSingleCameraNpyArrays:
- for video_number, this_synchronized_video_file_path in enumerate(synchronized_videos_path.glob('*.mp4')):
- this_video_capture_object = cv2.VideoCapture(str(this_synchronized_video_file_path))
- logger.info(f"Running `mediapipe` skeleton detection on video: {str(this_synchronized_video_file_path)}")
- this_video_mediapipe_payload_list = []
+ body_names_list = self._mediapipe_tracked_point_names_dict["body"]
+ right_hand_names_list = self._mediapipe_tracked_point_names_dict["right_hand"]
+ left_hand_names_list = self._mediapipe_tracked_point_names_dict["left_hand"]
+ face_names_list = self._mediapipe_tracked_point_names_dict["face"]
- success=True
- while success:
- success, image = this_video_capture_object.read()
- if image is None:
- logger.error(f"Failed to load an image from: {str(this_synchronized_video_file_path)}")
- raise Exception
- mediapipe_skeleton_payload = self.detect_skeleton_in_image(image, annotate_image=True)
- this_video_mediapipe_payload_list.append(mediapipe_skeleton_payload)
+ # apparently `mediapipe_results.pose_landmarks.landmark` returns something iterable Β―\_(ツ)_/Β―
+ mediapipe_pose_landmark_iterator = mp.python.solutions.pose.PoseLandmark
+ mediapipe_hand_landmark_iterator = mp.python.solutions.hands.HandLandmark
+ # TODO - build a better iterator and list of `face_marker_names` that will only pull out the face_counters & iris edges (mp.python.solutions.face_mesh_connections.FACEMESH_CONTOURS, FACE_MESH_IRISES)
- if save_annotated_videos:
- self.save_annotated_videos(this_video_mediapipe_payload_list, this_video_capture_object, this_synchronized_video_file_path.stem)
+ number_of_frames = len(mediapipe_results_list)
- def save_annotated_videos(self, mediapipe_payload_list:List, video_capture_object:cv2.VideoCapture, video_file_name:str):
+ number_of_body_trackedPoints = len(body_names_list)
+ number_of_right_hand_trackedPoints = len(right_hand_names_list)
+ number_of_left_hand_trackedPoints = len(left_hand_names_list)
+ number_of_face_trackedPoints = mp.python.solutions.face_mesh.FACEMESH_NUM_LANDMARKS_WITH_IRISES
+ number_of_spatial_dimensions = 2 # this will be 2d XY pixel data
- this_video_name = video_file_name + "_mediapipe.mp4"
- image_width = video_capture_object.get(cv2.CAP_PROP_FRAME_WIDTH)
- image_height= video_capture_object.get(cv2.CAP_PROP_FRAME_HEIGHT)
+ body2d_frameNumber_trackedPointNumber_XY = np.zeros(
+ (number_of_frames, number_of_body_trackedPoints, number_of_spatial_dimensions))
+ body2d_frameNumber_trackedPointNumber_XY[:] = np.nan
- video_recorder = VideoRecorder(this_video_name,
- image_width,
- image_height,
- self._session_id,
- mediapipe_annotated_video_bool=True)
+ body2d_frameNumber_trackedPointNumber_confidence = np.zeros((number_of_frames, number_of_body_trackedPoints))
+ body2d_frameNumber_trackedPointNumber_confidence[:] = np.nan # only body markers get a 'confidence' value
- image_list = [this_mediapipe_payload.annotated_image for this_mediapipe_payload in mediapipe_payload_list]
- logger.info(f'Saving mediapipe annotated video: {video_recorder.path_to_save_video_file}')
- video_recorder.save_image_list_to_disk(image_list, frame_rate=30)
+ rightHand2d_frameNumber_trackedPointNumber_XY = np.zeros((number_of_frames,
+ number_of_right_hand_trackedPoints,
+ number_of_spatial_dimensions))
+ rightHand2d_frameNumber_trackedPointNumber_XY[:] = np.nan
+
+ leftHand2d_frameNumber_trackedPointNumber_XY = np.zeros((number_of_frames,
+ number_of_left_hand_trackedPoints,
+ number_of_spatial_dimensions))
+ leftHand2d_frameNumber_trackedPointNumber_XY[:] = np.nan
+
+ face2d_frameNumber_trackedPointNumber_XY = np.zeros((number_of_frames,
+ number_of_face_trackedPoints,
+ number_of_spatial_dimensions))
+ face2d_frameNumber_trackedPointNumber_XY[:] = np.nan
+
+ all_body_tracked_points_visible_on_this_frame_bool_list = []
+ all_right_hand_points_visible_on_this_frame_bool_list = []
+ all_left_hand_points_visible_on_this_frame_bool_list = []
+ all_face_points_visible_on_this_frame_bool_list = []
+ all_tracked_points_visible_on_this_frame_list = []
+
+ for this_frame_number, this_frame_results in enumerate(mediapipe_results_list):
+
+ # get the Body data (aka 'pose')
+ if this_frame_results.pose_landmarks is not None:
+
+ for this_landmark_data in this_frame_results.pose_landmarks.landmark:
+ body2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,
+ 0] = this_landmark_data.x * image_width
+ body2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,
+ 1] = this_landmark_data.y * image_height
+ body2d_frameNumber_trackedPointNumber_confidence[this_frame_number,
+ :] = this_landmark_data.visibility # mediapipe calls their 'confidence' score 'visibility'
+
+ # get Right Hand data
+ if this_frame_results.right_hand_landmarks is not None:
+ for this_landmark_data in this_frame_results.right_hand_landmarks.landmark:
+ rightHand2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,
+ 0] = this_landmark_data.x * image_width
+ rightHand2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,
+ 1] = this_landmark_data.y * image_height
+
+ # get Left Hand data
+ if this_frame_results.left_hand_landmarks is not None:
+ for this_landmark_data in this_frame_results.left_hand_landmarks.landmark:
+ leftHand2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,
+ 0] = this_landmark_data.x * image_width
+ leftHand2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,
+ 1] = this_landmark_data.y * image_height
+
+ # get Face data
+ if this_frame_results.face_landmarks is not None:
+ for this_landmark_data in this_frame_results.face_landmarks.landmark:
+ face2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,
+ 0] = this_landmark_data.x * image_width
+ face2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,
+ 1] = this_landmark_data.y * image_height
+
+ # check if all tracked points are visible on this frame
+ all_body_visible = all(sum(
+ np.isnan(body2d_frameNumber_trackedPointNumber_XY[this_frame_number, :, :])) == 0)
+ all_body_tracked_points_visible_on_this_frame_bool_list.append(all_body_visible)
+
+ all_right_hand_visible = all(sum(
+ np.isnan(rightHand2d_frameNumber_trackedPointNumber_XY[this_frame_number, :, :])) == 0)
+ all_right_hand_points_visible_on_this_frame_bool_list.append(all_right_hand_visible)
+
+ all_left_hand_visible = all(sum(
+ np.isnan(leftHand2d_frameNumber_trackedPointNumber_XY[this_frame_number, :, :])) == 0)
+ all_left_hand_points_visible_on_this_frame_bool_list.append(all_left_hand_visible)
+ all_face_visible = all(sum(
+ np.isnan(face2d_frameNumber_trackedPointNumber_XY[this_frame_number, :, :])) == 0)
+ all_face_points_visible_on_this_frame_bool_list.append(all_face_visible)
+ all_points_visible = all([all_body_visible,
+ all_right_hand_visible,
+ all_left_hand_visible,
+ all_face_visible],
+ )
+ all_tracked_points_visible_on_this_frame_list.append(all_points_visible)
-class MediapipeSkeletonPayload(TweakedModel):
- mediapipe_results_list = List
- annotated_image = FramePayload
+ return Mediapipe2dSingleCameraNpyArrays(
+ body2d_frameNumber_trackedPointNumber_XY=body2d_frameNumber_trackedPointNumber_XY,
+ rightHand2d_frameNumber_trackedPointNumber_XY=rightHand2d_frameNumber_trackedPointNumber_XY,
+ leftHand2d_frameNumber_trackedPointNumber_XY=leftHand2d_frameNumber_trackedPointNumber_XY,
+ face2d_frameNumber_trackedPointNumber_XY=face2d_frameNumber_trackedPointNumber_XY,
+ body2d_frameNumber_trackedPointNumber_confidence=body2d_frameNumber_trackedPointNumber_confidence)
diff --git a/src/pipelines/calibration_pipeline/anipose_camera_calibration/anipose_camera_calibration.py b/src/pipelines/calibration_pipeline/anipose_camera_calibration/anipose_camera_calibration.py
index 8d641b4f..d482e8aa 100644
--- a/src/pipelines/calibration_pipeline/anipose_camera_calibration/anipose_camera_calibration.py
+++ b/src/pipelines/calibration_pipeline/anipose_camera_calibration/anipose_camera_calibration.py
@@ -6,7 +6,8 @@ import numpy as np
from cv2.cv2 import aruco_CharucoBoard
from scipy.spatial.transform import Rotation
-from src.config.home_dir import get_session_path, get_freemocap_data_folder_path
+from src.config.home_dir import get_session_folder_path, get_freemocap_data_folder_path, \
+ get_calibration_videos_folder_path
from src.pipelines.calibration_pipeline.anipose_camera_calibration import freemocap_anipose
from aniposelib.boards import CharucoBoard as AniposeCharucoBoard
@@ -34,8 +35,8 @@ class AniposeCameraCalibrator:
self.initialize_anipose_objects()
def get_paths_and_whatnot(self):
- self.session_folder_path = Path(get_session_path(self._session_id))
- calibration_videos_folder = self.session_folder_path / 'calibration_videos'
+ self.session_folder_path = Path(get_session_folder_path(self._session_id))
+ calibration_videos_folder = Path(get_calibration_videos_folder_path())
self._list_of_video_paths = [this_video_path for this_video_path in
calibration_videos_folder.glob('*.mp4'.lower())]
diff --git a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py
index 3c99868a..f5acf0b4 100644
--- a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py
+++ b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py
@@ -7,7 +7,7 @@ from typing import Union
import cv2
from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager
-from src.config.home_dir import create_session_id, get_session_path, get_freemocap_data_folder_path
+from src.config.home_dir import create_session_id, get_session_folder_path, get_freemocap_data_folder_path
from src.core_processor.fps.timestamp_manager import TimestampManager
from src.core_processor.show_cam_window import show_cam_window
from src.core_processor.utils.image_fps_writer import write_fps_to_image
@@ -42,7 +42,7 @@ class CalibrationPipelineOrchestrator:
@property
def session_folder_path(self):
- return get_session_path(self._session_id)
+ return get_session_folder_path(self._session_id)
def record_videos(
self,
@@ -153,7 +153,7 @@ if __name__ == "__main__":
record_new = True
if record_new:
session_id = create_session_id('calibration')
- session_path = Path(get_session_path(session_id))
+ session_path = Path(get_session_folder_path(session_id))
logger.info(f'Creating `calibration only` session folder at: {str(session_path)}')
session_path.mkdir(parents=True, exist_ok=False)
diff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
index 6aa3dcf2..1ce30c3a 100644
--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
@@ -10,7 +10,7 @@ import numpy as np
from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager
from src.cameras.persistence.video_writer.save_options_dataclass import SaveOptions
from src.config.data_paths import freemocap_data_path
-from src.config.home_dir import create_session_id, get_session_path
+from src.config.home_dir import create_session_id, get_session_folder_path
from src.core_processor.camera_calibration.camera_calibrator import CameraCalibrator
from src.core_processor.fps.timestamp_manager import TimestampManager
from src.core_processor.mediapipe_skeleton_detector.mediapipe_skeleton_detector import MediaPipeSkeletonDetector
@@ -306,6 +306,10 @@ class SessionPipelineOrchestrator:
data3d_trackedPointNum_xyz=data3d_trackedPointNum_xyz,
data3d_trackedPointNum_reprojectionError=data3d_trackedPointNum_reprojectionError, )
+ def mediapipe_track_skeletons(self):
+ self._mediapipe2d_numCams_numFrames_numTrackedPoints_XY = self.mediapipe_skeleton_detector.process_session_folder(save_annotated_videos=True)
+ pass
+
if __name__ == "__main__":
print('running `session_pipeline_orchestrator` as `__main__')
@@ -326,4 +330,4 @@ if __name__ == "__main__":
show_visualizer_gui=True,
)
- this_session_orchestrator.mediapipe_skeleton_detector.process_session_folder(save_annotated_videos=True)
+ this_session_orchestrator.mediapipe_track_skeletons()