Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

to make skreleton - run `session_router`, show charuco board to first videos, show humon body to second videos, wait -> πŸ’€βœ¨

Jon8b8188efcf79
diff --git a/src/api/routes/session/session_router.py b/src/api/routes/session/session_router.py
index 857e241f..b31cd4ed 100644
--- a/src/api/routes/session/session_router.py
+++ b/src/api/routes/session/session_router.py
@@ -11,6 +11,7 @@ from src.cameras.launch_camera_frame_loop import launch_camera_frame_loop
from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager
from src.config.home_dir import create_session_id, get_session_folder_path, get_most_recent_session_id, \
create_session_folder
+from src.core_processor.mediapipe_skeleton_detector.mediapipe_skeleton_detector import MediaPipeSkeletonDetector
from src.pipelines.calibration_pipeline.calibration_pipeline_orchestrator import CalibrationPipelineOrchestrator
from src.pipelines.session_pipeline.session_pipeline_orchestrator import SessionPipelineOrchestrator, \
load_mediapipe3d_skeleton_data, load_mediapipe2d_data
@@ -44,6 +45,10 @@ class SessionCalibrateModel(TweakedPydanticBaseModel):
opencv_camera_manager: OpenCVCameraManager = None
charuco_square_size: Union[int, float] = 39
+class SessionRecordModel(TweakedPydanticBaseModel):
+ session_id: str = None
+ webcam_configs_dict: dict = None
+
class SessionIdModel(BaseModel):
"""
@@ -98,10 +103,13 @@ def calibrate_session(session_calibrate_model: SessionCalibrateModel = SessionCa
@session_router.post("/session/record")
-def record_session(session_id_model: SessionIdModel = SessionIdModel()):
- this_session_orchestrator = SessionPipelineOrchestrator(session_id=session_id_model.session_id)
- this_session_orchestrator.record_new_session()
-
+def record_session(session_record_model: SessionRecordModel = SessionRecordModel()):
+ launch_camera_frame_loop(session_id=session_record_model.session_id,
+ webcam_configs_dict=session_record_model.webcam_configs_dict,
+ show_camera_views_in_windows=True,
+ calibration_videos_bool=False,
+ detect_charuco_in_image=True,
+ )
@session_router.post("/session/mediapipe_track_skeletons_offline")
def mediapipe_track_2D_skeletons_offline(session_id_model: SessionIdModel = SessionIdModel()):
@@ -112,8 +120,8 @@ def mediapipe_track_2D_skeletons_offline(session_id_model: SessionIdModel = Sess
this_session_id = session_id_model.session_id
logger.info(f"tracking 2D mediapipe skeletons in videos from session: {this_session_id}")
- this_session_orchestrator = SessionPipelineOrchestrator(session_id=this_session_id)
- this_session_orchestrator.mediapipe_track_skeletons_offline()
+ mediapipe_skeleton_detector = MediaPipeSkeletonDetector(this_session_id)
+ mediapipe_skeleton_detector.process_session_folder()
@session_router.post("/session/reconstruct_mediapipe3d_offline")
@@ -152,19 +160,19 @@ if __name__ == "__main__":
session_id_in = create_session_id('session_router_as_main')
session_id_model = SessionIdModel(session_id=session_id_in)
- # #calibrate_session
+ # # #calibrate_session
session_calibrate_model_in = SessionCalibrateModel(session_id=session_id_in,
charuco_square_size=39)
calibrate_session(session_calibrate_model_in)
# # record new session
- # record_session(session_id_model)
- #
- # # #process_
- # mediapipe_track_2D_skeletons_offline(session_id_model)
- # #
- # mediapipe_reconstruct_3D_skeletons_offline(session_id_model)
- #
- # # #visualize with PyQt/OpenGL
- #
- # visualize_session_offline(session_id_model)
+ session_record_model_in = SessionRecordModel(session_id=session_id_in,)
+ record_session(session_record_model_in)
+
+ # #process_
+ mediapipe_track_2D_skeletons_offline()
+
+ mediapipe_reconstruct_3D_skeletons_offline()
+
+ # #visualize with PyQt/OpenGL
+ visualize_session_offline()
diff --git a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
index 6f6e82e7..5191eb9e 100644
--- a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
+++ b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
@@ -6,7 +6,6 @@ from typing import List, Dict, Union, Any
import cv2
import numpy as np
import mediapipe as mp
-from rich.progress import Progress, track
from jon_scratch.opencv_camera import TweakedModel
from src.cameras.capture.dataclasses.frame_payload import FramePayload
@@ -141,8 +140,7 @@ class MediaPipeSkeletonDetector:
each_video_frame_height_list = []
mediapipe2d_single_camera_npy_arrays_list = []
- for video_number, this_synchronized_video_file_path in track(enumerate(synchronized_videos_path.glob('*.mp4')),
- description="detecting mediapipe skeletons"):
+ for video_number, this_synchronized_video_file_path in enumerate(synchronized_videos_path.glob('*.mp4')):
logger.info(f"Running `mediapipe` skeleton detection on video: {str(this_synchronized_video_file_path)}")
this_video_capture_object = cv2.VideoCapture(str(this_synchronized_video_file_path))
@@ -344,9 +342,9 @@ class MediaPipeSkeletonDetector:
# get Face data
if this_frame_results.face_landmarks is not None:
for this_landmark_number, this_landmark_data in enumerate(this_frame_results.face_landmarks.landmark):
- face2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,
+ face2d_frameNumber_trackedPointNumber_XY[this_frame_number, this_landmark_number,
0] = this_landmark_data.x * image_width
- face2d_frameNumber_trackedPointNumber_XY[this_frame_number, :,
+ face2d_frameNumber_trackedPointNumber_XY[this_frame_number, this_landmark_number,
1] = this_landmark_data.y * image_height
# check if all tracked points are visible on this frame
diff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
index 90b3372b..8ef06b44 100644
--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
@@ -46,8 +46,7 @@ class SessionPipelineOrchestrator:
self._expected_framerate = expected_framerate
self._visualizer_gui = QTVisualizerAndGui()
- self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id,
- expected_framerate=self.expected_framerate)
+ self._open_cv_camera_manager = None
self._charuco_board_detector = CharucoBoardDetector()
self._mediapipe_skeleton_detector = MediaPipeSkeletonDetector(self._session_id)
@@ -459,11 +458,11 @@ def load_mediapipe3d_skeleton_data(session_id: str = None):
output_data_folder = Path(get_session_output_data_folder_path(session_id))
mediapipe3d_xyz_file_path = output_data_folder / "mediapipe_3dData_numFrames_numTrackedPoints_spatialXYZ.npy"
logger.info(f"loading: {mediapipe3d_xyz_file_path}")
- mediapipe3d_skeleton_nFrames_nTrajectories_xyz = np.load(mediapipe3d_xyz_file_path)
+ mediapipe3d_skeleton_nFrames_nTrajectories_xyz = np.load(str(mediapipe3d_xyz_file_path))
mediapipe_reprojection_error_file_path = output_data_folder / "mediapipe_3dData_numFrames_numTrackedPoints_reprojectionError.npy"
logger.info(f"loading: {mediapipe_reprojection_error_file_path}")
- mediapipe3d_skeleton_nFrames_nTrajectories_reprojectionError = np.load(mediapipe_reprojection_error_file_path)
+ mediapipe3d_skeleton_nFrames_nTrajectories_reprojectionError = np.load(str(mediapipe_reprojection_error_file_path))
return Data3dFullSessionPayload(
data3d_numFrames_numTrackedPoints_XYZ=mediapipe3d_skeleton_nFrames_nTrajectories_xyz,