Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

say hello to mr skreleton (still pretty borked tho lol)

Jon2b7d9721de02
diff --git a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
index 7283dcda..5267d594 100644
--- a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
+++ b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
@@ -40,10 +40,10 @@ class Mediapipe2dNumpyArrays:
# if there's no body data, there's no hand or face data either
return
- return np.squeeze(np.hstack([self.body2d_frameNumber_trackedPointNumber_XY,
- self.rightHand2d_frameNumber_trackedPointNumber_XY,
- self.leftHand2d_frameNumber_trackedPointNumber_XY,
- self.face2d_frameNumber_trackedPointNumber_XY]))
+ return np.vstack([self.body2d_frameNumber_trackedPointNumber_XY,
+ self.rightHand2d_frameNumber_trackedPointNumber_XY,
+ self.leftHand2d_frameNumber_trackedPointNumber_XY,
+ self.face2d_frameNumber_trackedPointNumber_XY])
@dataclass
@@ -104,7 +104,16 @@ class MediaPipeSkeletonDetector:
annotated_image = self._annotate_image(annotated_image, mediapipe_results)
- mediapipe_single_frame_npy_data = self._list_of_mediapipe_results_to_npy_arrays([mediapipe_results])
+ mediapipe_single_frame_npy_data = self._list_of_mediapipe_results_to_npy_arrays([mediapipe_results],
+ image_width=
+ annotated_image.shape[0],
+ image_height=
+ annotated_image.shape[1])
+ mediapipe_single_frame_npy_data.body2d_frameNumber_trackedPointNumber_XY = self._threshold_by_confidence(
+ mediapipe_single_frame_npy_data.body2d_frameNumber_trackedPointNumber_XY,
+ mediapipe_single_frame_npy_data.body2d_frameNumber_trackedPointNumber_confidence,
+ confidence_threshold=.5)
+
return Mediapipe2dDataPayload(raw_frame_payload=raw_frame_payload,
mediapipe_results=mediapipe_results,
annotated_image=annotated_image,
@@ -239,8 +248,8 @@ class MediaPipeSkeletonDetector:
def _list_of_mediapipe_results_to_npy_arrays(self,
mediapipe_results_list: List,
- image_width: int = 1,
- image_height: int = 1
+ image_width: Union[int, float],
+ image_height: Union[int, float],
) -> Mediapipe2dNumpyArrays:
# apparently `mediapipe_results.pose_landmarks.landmark` returns something iterable Β―\_(ツ)_/Β―
@@ -350,8 +359,18 @@ class MediaPipeSkeletonDetector:
all_tracked_points_visible_on_this_frame_list.append(all_points_visible)
return Mediapipe2dNumpyArrays(
- body2d_frameNumber_trackedPointNumber_XY=body2d_frameNumber_trackedPointNumber_XY,
- rightHand2d_frameNumber_trackedPointNumber_XY=rightHand2d_frameNumber_trackedPointNumber_XY,
- leftHand2d_frameNumber_trackedPointNumber_XY=leftHand2d_frameNumber_trackedPointNumber_XY,
- face2d_frameNumber_trackedPointNumber_XY=face2d_frameNumber_trackedPointNumber_XY,
- body2d_frameNumber_trackedPointNumber_confidence=body2d_frameNumber_trackedPointNumber_confidence)
+ body2d_frameNumber_trackedPointNumber_XY=np.squeeze(body2d_frameNumber_trackedPointNumber_XY),
+ rightHand2d_frameNumber_trackedPointNumber_XY=np.squeeze(rightHand2d_frameNumber_trackedPointNumber_XY),
+ leftHand2d_frameNumber_trackedPointNumber_XY=np.squeeze(leftHand2d_frameNumber_trackedPointNumber_XY),
+ face2d_frameNumber_trackedPointNumber_XY=np.squeeze(face2d_frameNumber_trackedPointNumber_XY),
+ body2d_frameNumber_trackedPointNumber_confidence=np.squeeze(
+ body2d_frameNumber_trackedPointNumber_confidence))
+
+ def _threshold_by_confidence(self,
+ data2d_trackedPoint_dim: np.ndarray,
+ data2d_trackedPoint_confidence: np.ndarray,
+ confidence_threshold: float):
+
+ threshold_mask = data2d_trackedPoint_confidence < confidence_threshold
+ data2d_trackedPoint_dim[threshold_mask, :] = np.nan
+ return data2d_trackedPoint_dim
diff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
index d20a0226..0fd53966 100644
--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
@@ -15,7 +15,8 @@ from src.config.home_dir import create_session_id, get_session_folder_path, get_
get_most_recent_session_id, create_session_folder
from src.core_processor.camera_calibration.camera_calibrator import CameraCalibrator
from src.core_processor.timestamp_manager.timestamp_manager import TimestampManager
-from src.core_processor.mediapipe_skeleton_detector.mediapipe_skeleton_detector import MediaPipeSkeletonDetector
+from src.core_processor.mediapipe_skeleton_detector.mediapipe_skeleton_detector import MediaPipeSkeletonDetector, \
+ Mediapipe2dDataPayload
from src.core_processor.show_cam_window import show_cam_window
from src.core_processor.utils.image_fps_writer import write_fps_to_image
from src.debug_plot_makers.simple_plot_3d_points import simple_plot_3d
@@ -172,21 +173,22 @@ class SessionPipelineOrchestrator:
should_continue = True
while should_continue: # BIG FRAME LOOP STARTS HERE
- if detect_charuco:
- if len(dictionary_of_charuco_frame_payloads_on_this_multi_frame) > 0:
- charuco3d_multi_frame_payload = self._reconstruct_3d_charuco(
- dictionary_of_charuco_frame_payloads_on_this_multi_frame)
- if show_visualizer_gui and charuco3d_multi_frame_payload is not None:
- self._visualizer_gui.update_charuco_3d_dottos(charuco3d_multi_frame_payload)
- dictionary_of_charuco_frame_payloads_on_this_multi_frame = {}
-
- if detect_mediapipe:
- if len(dictionary_of_mediapipe_payloads_on_this_multi_frame) > 0:
- mediapipe3d_multi_frame_payload = self._reconstruct_3d_mediapipe(
- dictionary_of_mediapipe_payloads_on_this_multi_frame)
- if show_visualizer_gui and mediapipe3d_multi_frame_payload is not None:
- self._visualizer_gui.update_medaiapipe3d_skeleton(mediapipe3d_multi_frame_payload)
- dictionary_of_mediapipe_payloads_on_this_multi_frame = {}
+ if reconstruct_3d:
+ if detect_charuco:
+ if len(dictionary_of_charuco_frame_payloads_on_this_multi_frame) > 0:
+ charuco3d_multi_frame_payload = self._reconstruct_3d_charuco(
+ dictionary_of_charuco_frame_payloads_on_this_multi_frame)
+ if show_visualizer_gui and charuco3d_multi_frame_payload is not None:
+ self._visualizer_gui.update_charuco_3d_dottos(charuco3d_multi_frame_payload)
+ dictionary_of_charuco_frame_payloads_on_this_multi_frame = {}
+
+ if detect_mediapipe:
+ if len(dictionary_of_mediapipe_payloads_on_this_multi_frame) > 0:
+ mediapipe3d_multi_frame_payload = self._reconstruct_3d_mediapipe(
+ dictionary_of_mediapipe_payloads_on_this_multi_frame)
+ if show_visualizer_gui and mediapipe3d_multi_frame_payload is not None:
+ self._visualizer_gui.update_medaiapipe3d_skeleton(mediapipe3d_multi_frame_payload)
+ dictionary_of_mediapipe_payloads_on_this_multi_frame = {}
timestamp_manager.log_new_timestamp_for_main_loop_ns(time.perf_counter_ns())
@@ -350,6 +352,9 @@ class SessionPipelineOrchestrator:
if sum(is_there_any_mediapipe_data) >= min_cameras_to_reconstruct: # if at least 2 cameras have any data, then we can try to make some 3d dottos
mediapipe2d_data_per_cam_dict = {}
for this_webcam_id, this_mediapipe_payload in dictionary_of_mediapipe_payloads_on_this_multi_frame.items():
+
+ self._threshold_2d_data_by_confidence(this_mediapipe_payload.pixel_data_numpy_arrays, .5)
+
mediapipe2d_data_per_cam_dict[this_webcam_id] = this_mediapipe_payload.pixel_data_numpy_arrays.all_data2d_nFrames_nTrackedPts_XY
mediapipe_3d_data_payload = self._triangulate_2d_data(mediapipe2d_data_per_cam_dict,
@@ -441,6 +446,9 @@ class SessionPipelineOrchestrator:
logger.info(f"saving: {self._mediapipe_reprojection_error_save_path}")
np.save(str(self._mediapipe_reprojection_error_save_path), data3d_numFrames_numTrackedPoints_reprojectionError)
+ def _threshold_2d_data_by_confidence(self, mediapipe2d_data_payload:Mediapipe2dDataPayload, confidence_threshold:float):
+ pass
+
def load_mediapipe3d_skeleton_data(session_id: str = None):
if session_id is None: