Free Motion Capture for Everyone πβ¨
TypeScript
Virtual cameras and MediaPipe hand fixes (#889)
Aaron Cherianddd7aad08cd6
diff --git a/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeView.tsx b/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeView.tsxindex 1c6d47d7..a9f93cdc 100644--- a/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeView.tsx+++ b/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeView.tsx@@ -8,6 +8,7 @@ import {detectCameras,selectCameras,selectConnectedCameras,+ selectFilterVirtualCameras,selectIsLoading,selectIsPaused,useAppDispatch,@@ -26,12 +27,13 @@ export const CameraConfigTreeView: React.FC = () => {const isLoading = useAppSelector(selectIsLoading);const connectedCameras = useAppSelector(selectConnectedCameras);const isPaused = useAppSelector(selectIsPaused);+ const filterVirtualCameras = useAppSelector(selectFilterVirtualCameras);useEffect(() => {if (isConnected && cameras.length === 0) {- dispatch(detectCameras({filterVirtual: true}));+ dispatch(detectCameras({filterVirtual: filterVirtualCameras}));}- }, [isConnected, cameras.length, dispatch]);+ }, [isConnected, cameras.length, dispatch, filterVirtualCameras]);return (<div className="camera-config-sidebar-panel flex flex-col flex-1 bg-middark br-2 p-1 min-h-0 order-1">diff --git a/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeViewHeader.tsx b/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeViewHeader.tsxindex 5ad9d069..4991c357 100644--- a/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeViewHeader.tsx+++ b/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeViewHeader.tsx@@ -8,7 +8,7 @@ import {pauseUnpauseCameras,} from "@/store/slices/cameras/cameras-thunks";import {autoApplyToggled, savedSettingsCleared} from "@/store/slices/cameras/cameras-slice";-import {selectAutoApply} from "@/store/slices/cameras/cameras-selectors";+import {selectAutoApply, selectFilterVirtualCameras} from "@/store/slices/cameras/cameras-selectors";import {useTranslation} from 'react-i18next';import IconButton from "@/components/ui-components/IconButton";@@ -26,13 +26,14 @@ export const CameraConfigTreeViewHeader: React.FC<CameraConfigTreeViewHeaderPropconst dispatch = useAppDispatch();const {t} = useTranslation();const isAutoApply = useAppSelector(selectAutoApply);+ const filterVirtualCameras = useAppSelector(selectFilterVirtualCameras);const [isActionInProgress, setIsActionInProgress] = React.useState(false);const handleRefreshCameras = async (e: React.MouseEvent): Promise<void> => {e.stopPropagation();setIsActionInProgress(true);try {- await dispatch(detectCameras({filterVirtual: true})).unwrap();+ await dispatch(detectCameras({filterVirtual: filterVirtualCameras})).unwrap();} catch (error) {console.error('Error detecting cameras:', error);} finally {diff --git a/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraHeaderActions.tsx b/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraHeaderActions.tsxindex 0070f34d..fceabc83 100644--- a/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraHeaderActions.tsx+++ b/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraHeaderActions.tsx@@ -7,7 +7,7 @@ import {detectCameras,pauseUnpauseCameras,} from "@/store/slices/cameras/cameras-thunks";-import {recommendExposureForAll, selectConnectedCameras, selectIsLoading, selectIsPaused} from "@/store/slices/cameras";+import {recommendExposureForAll, selectConnectedCameras, selectFilterVirtualCameras, selectIsLoading, selectIsPaused} from "@/store/slices/cameras";import {selectIsAnyRecording} from "@/store/slices/recording/recording-slice";import {useTranslation} from 'react-i18next';import IconButton from "@/components/ui-components/IconButton";@@ -22,6 +22,7 @@ export const CameraHeaderActions: React.FC = () => {const isPaused = useAppSelector(selectIsPaused);const connectedCameras = useAppSelector(selectConnectedCameras);const isRecording = useAppSelector(selectIsAnyRecording);+ const filterVirtualCameras = useAppSelector(selectFilterVirtualCameras);useEffect(() => {if (isStoppingCameras && connectedCameras.length === 0) {@@ -30,8 +31,8 @@ export const CameraHeaderActions: React.FC = () => {}, [isStoppingCameras, connectedCameras.length]);const handleDetect = useCallback(() => {- dispatch(detectCameras({filterVirtual: true}));- }, [dispatch]);+ dispatch(detectCameras({filterVirtual: filterVirtualCameras}));+ }, [dispatch, filterVirtualCameras]);const handleUpdate = useCallback(() => {dispatch(camerasConnectOrUpdate());diff --git a/freemocap-ui/src/components/ui-components/SettingsModal.tsx b/freemocap-ui/src/components/ui-components/SettingsModal.tsxindex cb30517f..52b2da8a 100644--- a/freemocap-ui/src/components/ui-components/SettingsModal.tsx+++ b/freemocap-ui/src/components/ui-components/SettingsModal.tsx@@ -3,6 +3,7 @@ import {useElectronIPC} from '@/services';import {useAppDispatch, useAppSelector} from '@/store';import {recordingDirectoryChanged, selectIsAnyRecording} from '@/store/slices/recording';import {activeRecordingBaseDirectoryChanged} from '@/store/slices/active-recording';+import {filterVirtualCamerasToggled, selectFilterVirtualCameras} from '@/store/slices/cameras';import {recordingsDirFromBaseFolder} from '@/utils/dataFolder';import ButtonSm from '@/components/ui-components/ButtonSm';import IconButton from '@/components/ui-components/IconButton';@@ -23,6 +24,7 @@ export const SettingsModal: React.FC<SettingsModalProps> = ({open, onClose}) =>const {isElectron, api} = useElectronIPC();const dispatch = useAppDispatch();const isRecording = useAppSelector(selectIsAnyRecording);+ const filterVirtualCameras = useAppSelector(selectFilterVirtualCameras);const {startTour} = useTutorial();const [baseFolder, setBaseFolder] = useState<string>('');@@ -203,6 +205,20 @@ export const SettingsModal: React.FC<SettingsModalProps> = ({open, onClose}) =>)}</div>+ {/* Cameras */}+ <div className="flex flex-col gap-1 bg-secondary p-2 br-1">+ <SubactionHeader text="Cameras" className="text-gray"/>+ <Checkbox+ label="Include virtual cameras when detecting devices"+ checked={!filterVirtualCameras}+ onChange={() => dispatch(filterVirtualCamerasToggled())}+ />+ <p className="text sm text-darkgray">+ Most users should leave this off β virtual cameras (OBS, Zoom, etc.) are usually noise.+ Enable it if your real camera is a virtual device.+ </p>+ </div>+{/* Getting started */}<div className="flex flex-col gap-2 bg-secondary p-2 br-1"><SubactionHeader text="Getting started" className="text-gray"/>diff --git a/freemocap-ui/src/components/ui-components/WelcomeModal.tsx b/freemocap-ui/src/components/ui-components/WelcomeModal.tsxindex 489b37ca..71ff0248 100644--- a/freemocap-ui/src/components/ui-components/WelcomeModal.tsx+++ b/freemocap-ui/src/components/ui-components/WelcomeModal.tsx@@ -7,7 +7,7 @@ import { LanguageSwitcher } from '@/components/languages/LanguageSwitcher';import { VersionChip } from '@/components/ui-components/VersionChip';import { useAppDispatch, useAppSelector } from '@/store';import { camerasConnectOrUpdate, detectCameras } from '@/store/slices/cameras/cameras-thunks';-import { selectCameras } from '@/store/slices/cameras/cameras-selectors';+import { selectCameras, selectFilterVirtualCameras } from '@/store/slices/cameras/cameras-selectors';import { EXTERNAL_URLS } from '@/constants/external-urls';import ButtonSm from '@/components/ui-components/ButtonSm';import ButtonCard from '@/components/ui-components/ButtonCard';@@ -29,6 +29,7 @@ export const WelcomeModal: React.FC<WelcomeModalProps> = ({ open, onClose }) =>const { isElectron, api } = useElectronIPC();const { connectedCameraIds } = useServer();const cameras = useAppSelector(selectCameras);+ const filterVirtualCameras = useAppSelector(selectFilterVirtualCameras);const { startTour } = useTutorial();const prevCountRef = useRef(connectedCameraIds.length);@@ -82,13 +83,13 @@ export const WelcomeModal: React.FC<WelcomeModalProps> = ({ open, onClose }) =>onClose();if (cameras.length === 0) {try {- await dispatch(detectCameras({ filterVirtual: true })).unwrap();+ await dispatch(detectCameras({ filterVirtual: filterVirtualCameras })).unwrap();} catch {return;}}dispatch(camerasConnectOrUpdate());- }, [navigate, onClose, dispatch, cameras.length]);+ }, [navigate, onClose, dispatch, cameras.length, filterVirtualCameras]);const handleGoToPlayback = useCallback(() => {navigate('/playback');diff --git a/freemocap-ui/src/store/persistence-listener.ts b/freemocap-ui/src/store/persistence-listener.tsindex 4c6f131b..84c3570d 100644--- a/freemocap-ui/src/store/persistence-listener.ts+++ b/freemocap-ui/src/store/persistence-listener.ts@@ -74,6 +74,17 @@ persistenceListenerMiddleware.startListening({},});+// βββ cameras.filterVirtualCameras βββββββββββββββββββββββββββββββββββββββββββββ+persistenceListenerMiddleware.startListening({+ predicate: (_, curr, prev) =>+ (curr as RootState).cameras.filterVirtualCameras !== (prev as RootState).cameras.filterVirtualCameras,+ effect: async (_, api) => {+ api.cancelActiveListeners();+ await api.delay(DEBOUNCE_MS);+ saveToStorage('cameras.filterVirtualCameras', (api.getState() as RootState).cameras.filterVirtualCameras);+ },+});+// βββ blender.settings βββββββββββββββββββββββββββββββββββββββββββββββββββββββββpersistenceListenerMiddleware.startListening({predicate: (_, curr, prev) => {diff --git a/freemocap-ui/src/store/slices/cameras/cameras-selectors.ts b/freemocap-ui/src/store/slices/cameras/cameras-selectors.tsindex bdb8db47..4adfe54c 100644--- a/freemocap-ui/src/store/slices/cameras/cameras-selectors.ts+++ b/freemocap-ui/src/store/slices/cameras/cameras-selectors.ts@@ -8,6 +8,7 @@ export const selectCameras = (state: RootState) => state.cameras.cameras;export const selectIsPaused = (state: RootState) => state.cameras.isPaused;export const selectIsLoading = (state: RootState) => state.cameras.isLoading;export const selectAutoApply = (state: RootState) => state.cameras.autoApply;+export const selectFilterVirtualCameras = (state: RootState) => state.cameras.filterVirtualCameras;export const selectError = (state: RootState) => state.cameras.error;// ========== Derived Selectors ==========diff --git a/freemocap-ui/src/store/slices/cameras/cameras-slice.ts b/freemocap-ui/src/store/slices/cameras/cameras-slice.tsindex 3b8c4630..0e9d1d44 100644--- a/freemocap-ui/src/store/slices/cameras/cameras-slice.ts+++ b/freemocap-ui/src/store/slices/cameras/cameras-slice.ts@@ -16,6 +16,7 @@ import {PersistedCameraSettingsMap,savePersistedCameraSettings,} from './camera-settings-storage';+import {loadFromStorage} from '../../persistence';// Persist all current camera desired configs + selection to localStoragefunction persistAllCameraSettings(state: CamerasState): void {@@ -31,6 +32,7 @@ const initialState: CamerasState = {isPaused: false,isLoading: false,autoApply: true,+ filterVirtualCameras: loadFromStorage('cameras.filterVirtualCameras', true),error: null,};@@ -111,6 +113,10 @@ export const cameraSlice = createSlice({state.autoApply = !state.autoApply;},+ filterVirtualCamerasToggled: (state) => {+ state.filterVirtualCameras = !state.filterVirtualCameras;+ },+savedSettingsCleared: (state) => {clearPersistedCameraSettings();for (const camera of state.cameras) {@@ -237,5 +243,6 @@ export const {configCopiedToAll,recommendExposureForAll,autoApplyToggled,+ filterVirtualCamerasToggled,savedSettingsCleared,} = cameraSlice.actions;diff --git a/freemocap-ui/src/store/slices/cameras/cameras-thunks.ts b/freemocap-ui/src/store/slices/cameras/cameras-thunks.tsindex 9629d8f3..f014b202 100644--- a/freemocap-ui/src/store/slices/cameras/cameras-thunks.ts+++ b/freemocap-ui/src/store/slices/cameras/cameras-thunks.ts@@ -30,10 +30,15 @@ export const detectCameras = createAsyncThunk<>('cameras/detect',async (request = { filterVirtual: true }) => {- const response = await fetch(serverUrls.endpoints.detectCameras, {+ const params = new URLSearchParams();+ if (request.filterVirtual !== undefined) {+ params.set('filter_virtual', String(request.filterVirtual));+ }+ const query = params.toString();+ const url = query ? `${serverUrls.endpoints.detectCameras}?${query}` : serverUrls.endpoints.detectCameras;++ const response = await fetch(url, {method: 'POST',- headers: { 'Content-Type': 'application/json' },- body: JSON.stringify(request),});if (!response.ok) {diff --git a/freemocap-ui/src/store/slices/cameras/cameras-types.ts b/freemocap-ui/src/store/slices/cameras/cameras-types.tsindex 2d815e59..02eba692 100644--- a/freemocap-ui/src/store/slices/cameras/cameras-types.ts+++ b/freemocap-ui/src/store/slices/cameras/cameras-types.ts@@ -95,6 +95,7 @@ export interface CamerasState {isPaused: boolean;isLoading: boolean;autoApply: boolean;+ filterVirtualCameras: boolean;error: string | null;}diff --git a/freemocap/core/pipeline/posthoc/video_node.py b/freemocap/core/pipeline/posthoc/video_node.pyindex c9877339..b158ca63 100644--- a/freemocap/core/pipeline/posthoc/video_node.py+++ b/freemocap/core/pipeline/posthoc/video_node.py@@ -97,12 +97,14 @@ def _build_tracker(tracker_config: TrackerConfig) -> tuple[Tracker, object]:if _is_mediapipe_config(tracker_config):import skellytracker.core.detectors.keypoint_detectors.mediapipe # noqa: F401 (registry)+ from skellytracker.core.sessions.cpu_session import CpuSession, CpuSessionConfigfrom skellytracker.core.sessions.mediapipe_session import (MediaPipeSession,MediaPipeSessionConfig,)session = MediaPipeSession.create(MediaPipeSessionConfig())- tracker = Tracker.create(tracker_config, {"mediapipe": session})+ cpu_session = CpuSession.create(CpuSessionConfig())+ tracker = Tracker.create(tracker_config, {"mediapipe": session, "cpu": cpu_session})return tracker, sessionmodel_name = "rtmw-x-l_256x192"@@ -628,4 +630,8 @@ def _get_observation(observation = cache[frame_number]return observation, state- return tracker.process_image(image, frame_number, state)+ from freemocap.core.tracking.tracker_factory import merge_mediapipe_hand_face_children++ observation, state = tracker.process_image(image, frame_number, state)+ merge_mediapipe_hand_face_children(observation)+ return observation, statediff --git a/freemocap/core/pipeline/realtime/realtime_skeleton_inference_node.py b/freemocap/core/pipeline/realtime/realtime_skeleton_inference_node.pyindex 6181078a..2e42c0d0 100644--- a/freemocap/core/pipeline/realtime/realtime_skeleton_inference_node.py+++ b/freemocap/core/pipeline/realtime/realtime_skeleton_inference_node.py@@ -61,6 +61,7 @@ from freemocap.core.pipeline.abcs.pipeline_ipc import PipelineIPCfrom freemocap.core.pipeline.abcs.source_node_abc import SourceNodefrom freemocap.core.pipeline.realtime.realtime_pipeline_config import RealtimePipelineConfigfrom freemocap.core.pipeline.pipeline_stage_timer import PipelineStageTimer+from freemocap.core.tracking.tracker_factory import merge_mediapipe_hand_face_childrenfrom freemocap.core.types.type_overloads import TopicPublicationQueuefrom freemocap.pubsub.pubsub_manager import PubSubTopicManagerfrom freemocap.pubsub.pubsub_topics import (@@ -285,6 +286,7 @@ class RealtimeSkeletonInferenceNode(SourceNode):conf_threshold = pipeline_config.camera_node_config.confidence_thresholdper_camera_skeleton: dict[CameraIdString, Observation | None] = {}for camera_id, obs in observations.items():+ merge_mediapipe_hand_face_children(obs)body_stage = obs.stages.get("body")if body_stage is not None and body_stage.keypoints is not None:kpts = body_stage.keypointsdiff --git a/freemocap/core/tasks/mocap/mocap_task_config.py b/freemocap/core/tasks/mocap/mocap_task_config.pyindex 9db9abab..3eb219fb 100644--- a/freemocap/core/tasks/mocap/mocap_task_config.py+++ b/freemocap/core/tasks/mocap/mocap_task_config.py@@ -2,15 +2,6 @@ from typing import Literalfrom pydantic import BaseModel, ConfigDict, Field, model_validatorfrom skellytracker.core import DetectionStageConfig, TrackerConfig-from skellytracker.core.detectors.keypoint_detectors.mediapipe.body.mediapipe_pose_detector import (- MediapipePoseDetectorConfig,-)-from skellytracker.core.detectors.keypoint_detectors.mediapipe.face.mediapipe_face_detector import (- MediapipeFaceDetectorConfig,-)-from skellytracker.core.detectors.keypoint_detectors.mediapipe.hands.mediapipe_hand_detector import (- MediapipeHandDetectorConfig,-)from skellytracker.core.detectors.keypoint_detectors.mediapipe.mediapipe_model_manager import (MediapipePoseModelComplexity,)@@ -24,6 +15,7 @@ from skellytracker.core.temporal_processing.temporal_processing_config import (from skellyforge.post_processing.filters.filter_config import FilterConfigfrom freemocap.core.tasks.triangulation.helpers.triangulation_config import TriangulationConfig+from freemocap.core.tracking.tracker_factory import build_mediapipe_tracker_config@@ -133,33 +125,12 @@ class PosthocMocapPipelineConfig(BaseModel):@model_validator(mode="after")def _build_tracker_config_from_detector_type(self) -> "PosthocMocapPipelineConfig":if self.detector_type == "mediapipe":- self.tracker_config = TrackerConfig(- stages=[- DetectionStageConfig(- name="body",- keypoint_detectors=[- MediapipePoseDetectorConfig(- model_complexity=self.mediapipe_model_complexity,- num_poses=1,- min_pose_detection_confidence=self.mediapipe_detection_confidence,- min_pose_presence_confidence=self.mediapipe_presence_confidence,- min_pose_tracking_confidence=self.mediapipe_tracking_confidence,- ),- MediapipeHandDetectorConfig(- num_hands=self.mediapipe_num_hands,- min_hand_detection_confidence=self.mediapipe_detection_confidence,- min_hand_presence_confidence=self.mediapipe_presence_confidence,- min_hand_tracking_confidence=self.mediapipe_tracking_confidence,- ),- MediapipeFaceDetectorConfig(- num_faces=self.mediapipe_num_faces,- min_face_detection_confidence=self.mediapipe_detection_confidence,- min_face_presence_confidence=self.mediapipe_presence_confidence,- min_face_tracking_confidence=self.mediapipe_tracking_confidence,- ),- ],- )- ]+ self.tracker_config = build_mediapipe_tracker_config(+ model_complexity=self.mediapipe_model_complexity,+ detection_confidence=self.mediapipe_detection_confidence,+ presence_confidence=self.mediapipe_presence_confidence,+ tracking_confidence=self.mediapipe_tracking_confidence,+ num_faces=self.mediapipe_num_faces,)else:redetect_interval = max(1, round(5.0 * self.video_fps))diff --git a/freemocap/core/tracking/tracker_factory.py b/freemocap/core/tracking/tracker_factory.pyindex b446dc60..67b889c5 100644--- a/freemocap/core/tracking/tracker_factory.py+++ b/freemocap/core/tracking/tracker_factory.py@@ -18,6 +18,8 @@ import skellytracker.core.detectors.object_detectors.yolox # noqa: F401 (from skellytracker.core import (DetectionStageConfig,+ Keypoints,+ Observation,Tracker,TrackerConfig,)@@ -28,6 +30,9 @@ from skellytracker.core.detectors.keypoint_detectors.charuco import (from skellytracker.core.detectors.keypoint_detectors.rtmpose import (RTMPoseDetectorConfig,)+from skellytracker.core.detectors.object_detectors.keypoint_bbox import (+ KeypointBoundingBoxDetectorConfig,+)from skellytracker.core.detectors.object_detectors.yolox import (YoloxPersonDetector,YoloxPersonDetectorConfig,@@ -143,19 +148,66 @@ def build_skeleton_tracker(return Tracker.create(config, {"onnx": onnx_session})-def build_mediapipe_tracker(+def _mediapipe_hand_child_stage(+ side: str,+ *,+ detection_confidence: float,+ presence_confidence: float,+ tracking_confidence: float,+) -> DetectionStageConfig:+ """Build a wrist-cropped hand child stage for the given side ('left'/'right').++ Crops tightly around the wrist using the parent body stage's own+ wrist/index/pinky keypoints (pure keypoint arithmetic, no inference β see+ KeypointBoundingBoxDetectorConfig), then runs MediaPipe's hand landmarker+ on that zoomed-in crop with a fixed handedness, since MediaPipe's own+ per-crop handedness guess is unreliable on a tight single-hand crop.+ """+ from skellytracker.core.detectors.keypoint_detectors.mediapipe.hands.mediapipe_hand_detector import (+ MediapipeHandDetectorConfig,+ )++ return DetectionStageConfig(+ name=f"{side}_hand",+ object_detector=KeypointBoundingBoxDetectorConfig(+ center_keypoint_names=(f"{side}_index", f"{side}_pinky"),+ scale_keypoint_pairs=(+ (f"{side}_wrist", f"{side}_index"),+ (f"{side}_wrist", f"{side}_pinky"),+ ),+ scale_factor=3.0,+ min_box_size_px=120.0,+ ),+ keypoint_detectors=[+ MediapipeHandDetectorConfig(+ num_hands=1,+ assumed_handedness=side,+ min_hand_detection_confidence=detection_confidence,+ min_hand_presence_confidence=presence_confidence,+ min_hand_tracking_confidence=tracking_confidence,+ )+ ],+ bbox_smoothing=BBoxSmoothingConfig(alpha=0.4),+ )+++def build_mediapipe_tracker_config(*,model_complexity=None,detection_confidence: float = 0.5,presence_confidence: float = 0.5,tracking_confidence: float = 0.5,- num_hands: int = 2,num_faces: int = 1,-) -> tuple[Tracker, object]:- """Build a MediaPipe body+hands+face Tracker backed by a MediaPipeSession.+) -> TrackerConfig:+ """Build a MediaPipe body+hands+face TrackerConfig.- Returns (tracker, session). The session type is MediaPipeSession from- skellytracker.core.sessions.mediapipe_session.+ Pose is the parent "body" stage; left_hand/right_hand are wrist-cropped+ child stages (see _mediapipe_hand_child_stage) instead of full-frame+ sibling detectors, since hands occupy a tiny fraction of a full-body frame+ and MediaPipe's palm detector struggles to find them at that scale. Face+ is also a child stage (full-frame, no crop) purely to preserve the+ existing pose/right_hand/left_hand/face point ordering that+ merge_mediapipe_hand_face_children() and tracker_definitions.py expect."""from skellytracker.core.detectors.keypoint_detectors.mediapipe.body.mediapipe_pose_detector import (MediapipePoseDetectorConfig,@@ -163,22 +215,14 @@ def build_mediapipe_tracker(from skellytracker.core.detectors.keypoint_detectors.mediapipe.face.mediapipe_face_detector import (MediapipeFaceDetectorConfig,)- from skellytracker.core.detectors.keypoint_detectors.mediapipe.hands.mediapipe_hand_detector import (- MediapipeHandDetectorConfig,- )from skellytracker.core.detectors.keypoint_detectors.mediapipe.mediapipe_model_manager import (MediapipePoseModelComplexity,)- from skellytracker.core.sessions.mediapipe_session import (- MediaPipeSession,- MediaPipeSessionConfig,- )if model_complexity is None:model_complexity = MediapipePoseModelComplexity.HEAVY- session = MediaPipeSession.create(MediaPipeSessionConfig())- config = TrackerConfig(+ return TrackerConfig(stages=[DetectionStageConfig(name="body",@@ -190,22 +234,92 @@ def build_mediapipe_tracker(min_pose_presence_confidence=presence_confidence,min_pose_tracking_confidence=tracking_confidence,),- MediapipeHandDetectorConfig(- num_hands=num_hands,- min_hand_detection_confidence=detection_confidence,- min_hand_presence_confidence=presence_confidence,- min_hand_tracking_confidence=tracking_confidence,+ ],+ children=[+ _mediapipe_hand_child_stage(+ "right",+ detection_confidence=detection_confidence,+ presence_confidence=presence_confidence,+ tracking_confidence=tracking_confidence,+ ),+ _mediapipe_hand_child_stage(+ "left",+ detection_confidence=detection_confidence,+ presence_confidence=presence_confidence,+ tracking_confidence=tracking_confidence,),- MediapipeFaceDetectorConfig(- num_faces=num_faces,- min_face_detection_confidence=detection_confidence,- min_face_presence_confidence=presence_confidence,- min_face_tracking_confidence=tracking_confidence,+ DetectionStageConfig(+ name="face",+ keypoint_detectors=[+ MediapipeFaceDetectorConfig(+ num_faces=num_faces,+ min_face_detection_confidence=detection_confidence,+ min_face_presence_confidence=presence_confidence,+ min_face_tracking_confidence=tracking_confidence,+ )+ ],),],keypoint_reset_policy=KeypointResetPolicyConfig(max_consecutive_misses=10),)])- tracker = Tracker.create(config, {"mediapipe": session})+++def merge_mediapipe_hand_face_children(obs: Observation) -> None:+ """Merge left_hand/right_hand/face child-stage keypoints back into the+ top-level "body" stage, preserving the flat right_hand_*/left_hand_*+ naming and pose+right_hand+left_hand+face ordering every downstream+ consumer (skeleton_rigidifier, recording_status, export_to_blender,+ tracker_definitions, freemocap-ui) expects. Mutates `obs` in place.+ No-op for non-mediapipe configs (no "body" stage or no children).+ """+ body_stage = obs.stages.get("body")+ if body_stage is None or not body_stage.children:+ return+ parts = [body_stage.keypoints] if body_stage.keypoints is not None else []+ for name in ("right_hand", "left_hand", "face"):+ child = body_stage.children.pop(name, None)+ if child is not None and child.keypoints is not None:+ parts.append(child.keypoints)+ if len(parts) > 1:+ body_stage.keypoints = Keypoints.concatenate(parts)+++def build_mediapipe_tracker(+ *,+ model_complexity=None,+ detection_confidence: float = 0.5,+ presence_confidence: float = 0.5,+ tracking_confidence: float = 0.5,+ num_hands: int = 2,+ num_faces: int = 1,+) -> tuple[Tracker, object]:+ """Build a MediaPipe body+hands+face Tracker backed by a MediaPipeSession.++ Returns (tracker, session). The session type is MediaPipeSession from+ skellytracker.core.sessions.mediapipe_session.++ `num_hands` is accepted for API compatibility but unused: each hand child+ stage is a dedicated single-hand, wrist-cropped detector (see+ _mediapipe_hand_child_stage), so MediaPipe always looks for exactly one+ hand per stage rather than sharing a full-frame budget across both hands.+ """+ from skellytracker.core.sessions.mediapipe_session import (+ MediaPipeSession,+ MediaPipeSessionConfig,+ )++ session = MediaPipeSession.create(MediaPipeSessionConfig())+ config = build_mediapipe_tracker_config(+ model_complexity=model_complexity,+ detection_confidence=detection_confidence,+ presence_confidence=presence_confidence,+ tracking_confidence=tracking_confidence,+ num_faces=num_faces,+ )+ tracker = Tracker.create(+ config,+ {"mediapipe": session, "cpu": CpuSession.create(CpuSessionConfig())},+ )return tracker, sessiondiff --git a/uv.lock b/uv.lockindex 9c8da958..8c66da4b 100644--- a/uv.lock+++ b/uv.lock@@ -491,7 +491,7 @@ wheels = [[[package]]name = "freemocap"-version = "2.0.0a21"+version = "2.0.0a23"source = { editable = "." }dependencies = [{ name = "beartype" },@@ -2165,7 +2165,7 @@ dependencies = [[[package]]name = "skellycam"version = "2.0.0a6"-source = { git = "https://github.com/freemocap/skellycam?branch=main#d091ecef176a83bb1bc648bdf5f2ee702e2612da" }+source = { git = "https://github.com/freemocap/skellycam?branch=main#524e5fc513c99c72f20f76be0eccf7199b200dfb" }dependencies = [{ name = "av" },{ name = "beartype" },@@ -2223,7 +2223,7 @@ dependencies = [[[package]]name = "skellytracker"version = "2024.9.1019"-source = { git = "https://github.com/freemocap/skellytracker?branch=main#cb739aa5d059c90ba9c92ab5fbd987096423a638" }+source = { git = "https://github.com/freemocap/skellytracker?branch=main#63100055867b161d32fb1535ed73f73e1ddd187d" }dependencies = [{ name = "beartype" },{ name = "numpy" },