Free Motion Capture for Everyone πβ¨
TypeScript
to make skreleton - run `session_router`, show charuco board to first videos, show humon body to second videos, wait -> πβ¨
Jon8b8188efcf79
diff --git a/src/api/routes/session/session_router.py b/src/api/routes/session/session_router.pyindex 857e241f..b31cd4ed 100644--- a/src/api/routes/session/session_router.py+++ b/src/api/routes/session/session_router.py@@ -11,6 +11,7 @@ from src.cameras.launch_camera_frame_loop import launch_camera_frame_loopfrom src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManagerfrom src.config.home_dir import create_session_id, get_session_folder_path, get_most_recent_session_id, \create_session_folder+from src.core_processor.mediapipe_skeleton_detector.mediapipe_skeleton_detector import MediaPipeSkeletonDetectorfrom src.pipelines.calibration_pipeline.calibration_pipeline_orchestrator import CalibrationPipelineOrchestratorfrom src.pipelines.session_pipeline.session_pipeline_orchestrator import SessionPipelineOrchestrator, \load_mediapipe3d_skeleton_data, load_mediapipe2d_data@@ -44,6 +45,10 @@ class SessionCalibrateModel(TweakedPydanticBaseModel):opencv_camera_manager: OpenCVCameraManager = Nonecharuco_square_size: Union[int, float] = 39+class SessionRecordModel(TweakedPydanticBaseModel):+ session_id: str = None+ webcam_configs_dict: dict = None+class SessionIdModel(BaseModel):"""@@ -98,10 +103,13 @@ def calibrate_session(session_calibrate_model: SessionCalibrateModel = SessionCa@session_router.post("/session/record")-def record_session(session_id_model: SessionIdModel = SessionIdModel()):- this_session_orchestrator = SessionPipelineOrchestrator(session_id=session_id_model.session_id)- this_session_orchestrator.record_new_session()-+def record_session(session_record_model: SessionRecordModel = SessionRecordModel()):+ launch_camera_frame_loop(session_id=session_record_model.session_id,+ webcam_configs_dict=session_record_model.webcam_configs_dict,+ show_camera_views_in_windows=True,+ calibration_videos_bool=False,+ detect_charuco_in_image=True,+ )@session_router.post("/session/mediapipe_track_skeletons_offline")def mediapipe_track_2D_skeletons_offline(session_id_model: SessionIdModel = SessionIdModel()):@@ -112,8 +120,8 @@ def mediapipe_track_2D_skeletons_offline(session_id_model: SessionIdModel = Sessthis_session_id = session_id_model.session_idlogger.info(f"tracking 2D mediapipe skeletons in videos from session: {this_session_id}")- this_session_orchestrator = SessionPipelineOrchestrator(session_id=this_session_id)- this_session_orchestrator.mediapipe_track_skeletons_offline()+ mediapipe_skeleton_detector = MediaPipeSkeletonDetector(this_session_id)+ mediapipe_skeleton_detector.process_session_folder()@session_router.post("/session/reconstruct_mediapipe3d_offline")@@ -152,19 +160,19 @@ if __name__ == "__main__":session_id_in = create_session_id('session_router_as_main')session_id_model = SessionIdModel(session_id=session_id_in)- # #calibrate_session+ # # #calibrate_sessionsession_calibrate_model_in = SessionCalibrateModel(session_id=session_id_in,charuco_square_size=39)calibrate_session(session_calibrate_model_in)# # record new session- # record_session(session_id_model)- #- # # #process_- # mediapipe_track_2D_skeletons_offline(session_id_model)- # #- # mediapipe_reconstruct_3D_skeletons_offline(session_id_model)- #- # # #visualize with PyQt/OpenGL- #- # visualize_session_offline(session_id_model)+ session_record_model_in = SessionRecordModel(session_id=session_id_in,)+ record_session(session_record_model_in)++ # #process_+ mediapipe_track_2D_skeletons_offline()++ mediapipe_reconstruct_3D_skeletons_offline()++ # #visualize with PyQt/OpenGL+ visualize_session_offline()diff --git a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.pyindex 6f6e82e7..5191eb9e 100644--- a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py+++ b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py@@ -6,7 +6,6 @@ from typing import List, Dict, Union, Anyimport cv2import numpy as npimport mediapipe as mp-from rich.progress import Progress, trackfrom jon_scratch.opencv_camera import TweakedModelfrom src.cameras.capture.dataclasses.frame_payload import FramePayload@@ -141,8 +140,7 @@ class MediaPipeSkeletonDetector:each_video_frame_height_list = []mediapipe2d_single_camera_npy_arrays_list = []- for video_number, this_synchronized_video_file_path in track(enumerate(synchronized_videos_path.glob('*.mp4')),- description="detecting mediapipe skeletons"):+ for video_number, this_synchronized_video_file_path in enumerate(synchronized_videos_path.glob('*.mp4')):logger.info(f"Running `mediapipe` skeleton detection on video: {str(this_synchronized_video_file_path)}")this_video_capture_object = cv2.VideoCapture(str(this_synchronized_video_file_path))@@ -344,9 +342,9 @@ class MediaPipeSkeletonDetector:# get Face dataif this_frame_results.face_landmarks is not None:for this_landmark_number, this_landmark_data in enumerate(this_frame_results.face_landmarks.landmark):- face2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,+ face2d_frameNumber_trackedPointNumber_XY[this_frame_number, this_landmark_number,0] = this_landmark_data.x * image_width- face2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,+ face2d_frameNumber_trackedPointNumber_XY[this_frame_number, this_landmark_number,1] = this_landmark_data.y * image_height# check if all tracked points are visible on this framediff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.pyindex 90b3372b..8ef06b44 100644--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py@@ -46,8 +46,7 @@ class SessionPipelineOrchestrator:self._expected_framerate = expected_framerateself._visualizer_gui = QTVisualizerAndGui()- self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id,- expected_framerate=self.expected_framerate)+ self._open_cv_camera_manager = Noneself._charuco_board_detector = CharucoBoardDetector()self._mediapipe_skeleton_detector = MediaPipeSkeletonDetector(self._session_id)@@ -459,11 +458,11 @@ def load_mediapipe3d_skeleton_data(session_id: str = None):output_data_folder = Path(get_session_output_data_folder_path(session_id))mediapipe3d_xyz_file_path = output_data_folder / "mediapipe_3dData_numFrames_numTrackedPoints_spatialXYZ.npy"logger.info(f"loading: {mediapipe3d_xyz_file_path}")- mediapipe3d_skeleton_nFrames_nTrajectories_xyz = np.load(mediapipe3d_xyz_file_path)+ mediapipe3d_skeleton_nFrames_nTrajectories_xyz = np.load(str(mediapipe3d_xyz_file_path))mediapipe_reprojection_error_file_path = output_data_folder / "mediapipe_3dData_numFrames_numTrackedPoints_reprojectionError.npy"logger.info(f"loading: {mediapipe_reprojection_error_file_path}")- mediapipe3d_skeleton_nFrames_nTrajectories_reprojectionError = np.load(mediapipe_reprojection_error_file_path)+ mediapipe3d_skeleton_nFrames_nTrajectories_reprojectionError = np.load(str(mediapipe_reprojection_error_file_path))return Data3dFullSessionPayload(data3d_numFrames_numTrackedPoints_XYZ=mediapipe3d_skeleton_nFrames_nTrajectories_xyz,