Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

Virtual cameras and MediaPipe hand fixes (#889)

Aaron Cherianddd7aad08cd6
diff --git a/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeView.tsx b/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeView.tsx
index 1c6d47d7..a9f93cdc 100644
--- a/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeView.tsx
+++ b/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeView.tsx
@@ -8,6 +8,7 @@ import {
detectCameras,
selectCameras,
selectConnectedCameras,
+ selectFilterVirtualCameras,
selectIsLoading,
selectIsPaused,
useAppDispatch,
@@ -26,12 +27,13 @@ export const CameraConfigTreeView: React.FC = () => {
const isLoading = useAppSelector(selectIsLoading);
const connectedCameras = useAppSelector(selectConnectedCameras);
const isPaused = useAppSelector(selectIsPaused);
+ const filterVirtualCameras = useAppSelector(selectFilterVirtualCameras);
useEffect(() => {
if (isConnected && cameras.length === 0) {
- dispatch(detectCameras({filterVirtual: true}));
+ dispatch(detectCameras({filterVirtual: filterVirtualCameras}));
}
- }, [isConnected, cameras.length, dispatch]);
+ }, [isConnected, cameras.length, dispatch, filterVirtualCameras]);
return (
<div className="camera-config-sidebar-panel flex flex-col flex-1 bg-middark br-2 p-1 min-h-0 order-1">
diff --git a/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeViewHeader.tsx b/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeViewHeader.tsx
index 5ad9d069..4991c357 100644
--- a/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeViewHeader.tsx
+++ b/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraConfigTreeViewHeader.tsx
@@ -8,7 +8,7 @@ import {
pauseUnpauseCameras,
} from "@/store/slices/cameras/cameras-thunks";
import {autoApplyToggled, savedSettingsCleared} from "@/store/slices/cameras/cameras-slice";
-import {selectAutoApply} from "@/store/slices/cameras/cameras-selectors";
+import {selectAutoApply, selectFilterVirtualCameras} from "@/store/slices/cameras/cameras-selectors";
import {useTranslation} from 'react-i18next';
import IconButton from "@/components/ui-components/IconButton";
@@ -26,13 +26,14 @@ export const CameraConfigTreeViewHeader: React.FC<CameraConfigTreeViewHeaderProp
const dispatch = useAppDispatch();
const {t} = useTranslation();
const isAutoApply = useAppSelector(selectAutoApply);
+ const filterVirtualCameras = useAppSelector(selectFilterVirtualCameras);
const [isActionInProgress, setIsActionInProgress] = React.useState(false);
const handleRefreshCameras = async (e: React.MouseEvent): Promise<void> => {
e.stopPropagation();
setIsActionInProgress(true);
try {
- await dispatch(detectCameras({filterVirtual: true})).unwrap();
+ await dispatch(detectCameras({filterVirtual: filterVirtualCameras})).unwrap();
} catch (error) {
console.error('Error detecting cameras:', error);
} finally {
diff --git a/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraHeaderActions.tsx b/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraHeaderActions.tsx
index 0070f34d..fceabc83 100644
--- a/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraHeaderActions.tsx
+++ b/freemocap-ui/src/components/control-panels/camera-config-panel/camera-config-tree-view/CameraHeaderActions.tsx
@@ -7,7 +7,7 @@ import {
detectCameras,
pauseUnpauseCameras,
} from "@/store/slices/cameras/cameras-thunks";
-import {recommendExposureForAll, selectConnectedCameras, selectIsLoading, selectIsPaused} from "@/store/slices/cameras";
+import {recommendExposureForAll, selectConnectedCameras, selectFilterVirtualCameras, selectIsLoading, selectIsPaused} from "@/store/slices/cameras";
import {selectIsAnyRecording} from "@/store/slices/recording/recording-slice";
import {useTranslation} from 'react-i18next';
import IconButton from "@/components/ui-components/IconButton";
@@ -22,6 +22,7 @@ export const CameraHeaderActions: React.FC = () => {
const isPaused = useAppSelector(selectIsPaused);
const connectedCameras = useAppSelector(selectConnectedCameras);
const isRecording = useAppSelector(selectIsAnyRecording);
+ const filterVirtualCameras = useAppSelector(selectFilterVirtualCameras);
useEffect(() => {
if (isStoppingCameras && connectedCameras.length === 0) {
@@ -30,8 +31,8 @@ export const CameraHeaderActions: React.FC = () => {
}, [isStoppingCameras, connectedCameras.length]);
const handleDetect = useCallback(() => {
- dispatch(detectCameras({filterVirtual: true}));
- }, [dispatch]);
+ dispatch(detectCameras({filterVirtual: filterVirtualCameras}));
+ }, [dispatch, filterVirtualCameras]);
const handleUpdate = useCallback(() => {
dispatch(camerasConnectOrUpdate());
diff --git a/freemocap-ui/src/components/ui-components/SettingsModal.tsx b/freemocap-ui/src/components/ui-components/SettingsModal.tsx
index cb30517f..52b2da8a 100644
--- a/freemocap-ui/src/components/ui-components/SettingsModal.tsx
+++ b/freemocap-ui/src/components/ui-components/SettingsModal.tsx
@@ -3,6 +3,7 @@ import {useElectronIPC} from '@/services';
import {useAppDispatch, useAppSelector} from '@/store';
import {recordingDirectoryChanged, selectIsAnyRecording} from '@/store/slices/recording';
import {activeRecordingBaseDirectoryChanged} from '@/store/slices/active-recording';
+import {filterVirtualCamerasToggled, selectFilterVirtualCameras} from '@/store/slices/cameras';
import {recordingsDirFromBaseFolder} from '@/utils/dataFolder';
import ButtonSm from '@/components/ui-components/ButtonSm';
import IconButton from '@/components/ui-components/IconButton';
@@ -23,6 +24,7 @@ export const SettingsModal: React.FC<SettingsModalProps> = ({open, onClose}) =>
const {isElectron, api} = useElectronIPC();
const dispatch = useAppDispatch();
const isRecording = useAppSelector(selectIsAnyRecording);
+ const filterVirtualCameras = useAppSelector(selectFilterVirtualCameras);
const {startTour} = useTutorial();
const [baseFolder, setBaseFolder] = useState<string>('');
@@ -203,6 +205,20 @@ export const SettingsModal: React.FC<SettingsModalProps> = ({open, onClose}) =>
)}
</div>
+ {/* Cameras */}
+ <div className="flex flex-col gap-1 bg-secondary p-2 br-1">
+ <SubactionHeader text="Cameras" className="text-gray"/>
+ <Checkbox
+ label="Include virtual cameras when detecting devices"
+ checked={!filterVirtualCameras}
+ onChange={() => dispatch(filterVirtualCamerasToggled())}
+ />
+ <p className="text sm text-darkgray">
+ Most users should leave this off β€” virtual cameras (OBS, Zoom, etc.) are usually noise.
+ Enable it if your real camera is a virtual device.
+ </p>
+ </div>
+
{/* Getting started */}
<div className="flex flex-col gap-2 bg-secondary p-2 br-1">
<SubactionHeader text="Getting started" className="text-gray"/>
diff --git a/freemocap-ui/src/components/ui-components/WelcomeModal.tsx b/freemocap-ui/src/components/ui-components/WelcomeModal.tsx
index 489b37ca..71ff0248 100644
--- a/freemocap-ui/src/components/ui-components/WelcomeModal.tsx
+++ b/freemocap-ui/src/components/ui-components/WelcomeModal.tsx
@@ -7,7 +7,7 @@ import { LanguageSwitcher } from '@/components/languages/LanguageSwitcher';
import { VersionChip } from '@/components/ui-components/VersionChip';
import { useAppDispatch, useAppSelector } from '@/store';
import { camerasConnectOrUpdate, detectCameras } from '@/store/slices/cameras/cameras-thunks';
-import { selectCameras } from '@/store/slices/cameras/cameras-selectors';
+import { selectCameras, selectFilterVirtualCameras } from '@/store/slices/cameras/cameras-selectors';
import { EXTERNAL_URLS } from '@/constants/external-urls';
import ButtonSm from '@/components/ui-components/ButtonSm';
import ButtonCard from '@/components/ui-components/ButtonCard';
@@ -29,6 +29,7 @@ export const WelcomeModal: React.FC<WelcomeModalProps> = ({ open, onClose }) =>
const { isElectron, api } = useElectronIPC();
const { connectedCameraIds } = useServer();
const cameras = useAppSelector(selectCameras);
+ const filterVirtualCameras = useAppSelector(selectFilterVirtualCameras);
const { startTour } = useTutorial();
const prevCountRef = useRef(connectedCameraIds.length);
@@ -82,13 +83,13 @@ export const WelcomeModal: React.FC<WelcomeModalProps> = ({ open, onClose }) =>
onClose();
if (cameras.length === 0) {
try {
- await dispatch(detectCameras({ filterVirtual: true })).unwrap();
+ await dispatch(detectCameras({ filterVirtual: filterVirtualCameras })).unwrap();
} catch {
return;
}
}
dispatch(camerasConnectOrUpdate());
- }, [navigate, onClose, dispatch, cameras.length]);
+ }, [navigate, onClose, dispatch, cameras.length, filterVirtualCameras]);
const handleGoToPlayback = useCallback(() => {
navigate('/playback');
diff --git a/freemocap-ui/src/store/persistence-listener.ts b/freemocap-ui/src/store/persistence-listener.ts
index 4c6f131b..84c3570d 100644
--- a/freemocap-ui/src/store/persistence-listener.ts
+++ b/freemocap-ui/src/store/persistence-listener.ts
@@ -74,6 +74,17 @@ persistenceListenerMiddleware.startListening({
},
});
+// ─── cameras.filterVirtualCameras ─────────────────────────────────────────────
+persistenceListenerMiddleware.startListening({
+ predicate: (_, curr, prev) =>
+ (curr as RootState).cameras.filterVirtualCameras !== (prev as RootState).cameras.filterVirtualCameras,
+ effect: async (_, api) => {
+ api.cancelActiveListeners();
+ await api.delay(DEBOUNCE_MS);
+ saveToStorage('cameras.filterVirtualCameras', (api.getState() as RootState).cameras.filterVirtualCameras);
+ },
+});
+
// ─── blender.settings ─────────────────────────────────────────────────────────
persistenceListenerMiddleware.startListening({
predicate: (_, curr, prev) => {
diff --git a/freemocap-ui/src/store/slices/cameras/cameras-selectors.ts b/freemocap-ui/src/store/slices/cameras/cameras-selectors.ts
index bdb8db47..4adfe54c 100644
--- a/freemocap-ui/src/store/slices/cameras/cameras-selectors.ts
+++ b/freemocap-ui/src/store/slices/cameras/cameras-selectors.ts
@@ -8,6 +8,7 @@ export const selectCameras = (state: RootState) => state.cameras.cameras;
export const selectIsPaused = (state: RootState) => state.cameras.isPaused;
export const selectIsLoading = (state: RootState) => state.cameras.isLoading;
export const selectAutoApply = (state: RootState) => state.cameras.autoApply;
+export const selectFilterVirtualCameras = (state: RootState) => state.cameras.filterVirtualCameras;
export const selectError = (state: RootState) => state.cameras.error;
// ========== Derived Selectors ==========
diff --git a/freemocap-ui/src/store/slices/cameras/cameras-slice.ts b/freemocap-ui/src/store/slices/cameras/cameras-slice.ts
index 3b8c4630..0e9d1d44 100644
--- a/freemocap-ui/src/store/slices/cameras/cameras-slice.ts
+++ b/freemocap-ui/src/store/slices/cameras/cameras-slice.ts
@@ -16,6 +16,7 @@ import {
PersistedCameraSettingsMap,
savePersistedCameraSettings,
} from './camera-settings-storage';
+import {loadFromStorage} from '../../persistence';
// Persist all current camera desired configs + selection to localStorage
function persistAllCameraSettings(state: CamerasState): void {
@@ -31,6 +32,7 @@ const initialState: CamerasState = {
isPaused: false,
isLoading: false,
autoApply: true,
+ filterVirtualCameras: loadFromStorage('cameras.filterVirtualCameras', true),
error: null,
};
@@ -111,6 +113,10 @@ export const cameraSlice = createSlice({
state.autoApply = !state.autoApply;
},
+ filterVirtualCamerasToggled: (state) => {
+ state.filterVirtualCameras = !state.filterVirtualCameras;
+ },
+
savedSettingsCleared: (state) => {
clearPersistedCameraSettings();
for (const camera of state.cameras) {
@@ -237,5 +243,6 @@ export const {
configCopiedToAll,
recommendExposureForAll,
autoApplyToggled,
+ filterVirtualCamerasToggled,
savedSettingsCleared,
} = cameraSlice.actions;
diff --git a/freemocap-ui/src/store/slices/cameras/cameras-thunks.ts b/freemocap-ui/src/store/slices/cameras/cameras-thunks.ts
index 9629d8f3..f014b202 100644
--- a/freemocap-ui/src/store/slices/cameras/cameras-thunks.ts
+++ b/freemocap-ui/src/store/slices/cameras/cameras-thunks.ts
@@ -30,10 +30,15 @@ export const detectCameras = createAsyncThunk<
>(
'cameras/detect',
async (request = { filterVirtual: true }) => {
- const response = await fetch(serverUrls.endpoints.detectCameras, {
+ const params = new URLSearchParams();
+ if (request.filterVirtual !== undefined) {
+ params.set('filter_virtual', String(request.filterVirtual));
+ }
+ const query = params.toString();
+ const url = query ? `${serverUrls.endpoints.detectCameras}?${query}` : serverUrls.endpoints.detectCameras;
+
+ const response = await fetch(url, {
method: 'POST',
- headers: { 'Content-Type': 'application/json' },
- body: JSON.stringify(request),
});
if (!response.ok) {
diff --git a/freemocap-ui/src/store/slices/cameras/cameras-types.ts b/freemocap-ui/src/store/slices/cameras/cameras-types.ts
index 2d815e59..02eba692 100644
--- a/freemocap-ui/src/store/slices/cameras/cameras-types.ts
+++ b/freemocap-ui/src/store/slices/cameras/cameras-types.ts
@@ -95,6 +95,7 @@ export interface CamerasState {
isPaused: boolean;
isLoading: boolean;
autoApply: boolean;
+ filterVirtualCameras: boolean;
error: string | null;
}
diff --git a/freemocap/core/pipeline/posthoc/video_node.py b/freemocap/core/pipeline/posthoc/video_node.py
index c9877339..b158ca63 100644
--- a/freemocap/core/pipeline/posthoc/video_node.py
+++ b/freemocap/core/pipeline/posthoc/video_node.py
@@ -97,12 +97,14 @@ def _build_tracker(tracker_config: TrackerConfig) -> tuple[Tracker, object]:
if _is_mediapipe_config(tracker_config):
import skellytracker.core.detectors.keypoint_detectors.mediapipe # noqa: F401 (registry)
+ from skellytracker.core.sessions.cpu_session import CpuSession, CpuSessionConfig
from skellytracker.core.sessions.mediapipe_session import (
MediaPipeSession,
MediaPipeSessionConfig,
)
session = MediaPipeSession.create(MediaPipeSessionConfig())
- tracker = Tracker.create(tracker_config, {"mediapipe": session})
+ cpu_session = CpuSession.create(CpuSessionConfig())
+ tracker = Tracker.create(tracker_config, {"mediapipe": session, "cpu": cpu_session})
return tracker, session
model_name = "rtmw-x-l_256x192"
@@ -628,4 +630,8 @@ def _get_observation(
observation = cache[frame_number]
return observation, state
- return tracker.process_image(image, frame_number, state)
+ from freemocap.core.tracking.tracker_factory import merge_mediapipe_hand_face_children
+
+ observation, state = tracker.process_image(image, frame_number, state)
+ merge_mediapipe_hand_face_children(observation)
+ return observation, state
diff --git a/freemocap/core/pipeline/realtime/realtime_skeleton_inference_node.py b/freemocap/core/pipeline/realtime/realtime_skeleton_inference_node.py
index 6181078a..2e42c0d0 100644
--- a/freemocap/core/pipeline/realtime/realtime_skeleton_inference_node.py
+++ b/freemocap/core/pipeline/realtime/realtime_skeleton_inference_node.py
@@ -61,6 +61,7 @@ from freemocap.core.pipeline.abcs.pipeline_ipc import PipelineIPC
from freemocap.core.pipeline.abcs.source_node_abc import SourceNode
from freemocap.core.pipeline.realtime.realtime_pipeline_config import RealtimePipelineConfig
from freemocap.core.pipeline.pipeline_stage_timer import PipelineStageTimer
+from freemocap.core.tracking.tracker_factory import merge_mediapipe_hand_face_children
from freemocap.core.types.type_overloads import TopicPublicationQueue
from freemocap.pubsub.pubsub_manager import PubSubTopicManager
from freemocap.pubsub.pubsub_topics import (
@@ -285,6 +286,7 @@ class RealtimeSkeletonInferenceNode(SourceNode):
conf_threshold = pipeline_config.camera_node_config.confidence_threshold
per_camera_skeleton: dict[CameraIdString, Observation | None] = {}
for camera_id, obs in observations.items():
+ merge_mediapipe_hand_face_children(obs)
body_stage = obs.stages.get("body")
if body_stage is not None and body_stage.keypoints is not None:
kpts = body_stage.keypoints
diff --git a/freemocap/core/tasks/mocap/mocap_task_config.py b/freemocap/core/tasks/mocap/mocap_task_config.py
index 9db9abab..3eb219fb 100644
--- a/freemocap/core/tasks/mocap/mocap_task_config.py
+++ b/freemocap/core/tasks/mocap/mocap_task_config.py
@@ -2,15 +2,6 @@ from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, model_validator
from skellytracker.core import DetectionStageConfig, TrackerConfig
-from skellytracker.core.detectors.keypoint_detectors.mediapipe.body.mediapipe_pose_detector import (
- MediapipePoseDetectorConfig,
-)
-from skellytracker.core.detectors.keypoint_detectors.mediapipe.face.mediapipe_face_detector import (
- MediapipeFaceDetectorConfig,
-)
-from skellytracker.core.detectors.keypoint_detectors.mediapipe.hands.mediapipe_hand_detector import (
- MediapipeHandDetectorConfig,
-)
from skellytracker.core.detectors.keypoint_detectors.mediapipe.mediapipe_model_manager import (
MediapipePoseModelComplexity,
)
@@ -24,6 +15,7 @@ from skellytracker.core.temporal_processing.temporal_processing_config import (
from skellyforge.post_processing.filters.filter_config import FilterConfig
from freemocap.core.tasks.triangulation.helpers.triangulation_config import TriangulationConfig
+from freemocap.core.tracking.tracker_factory import build_mediapipe_tracker_config
@@ -133,33 +125,12 @@ class PosthocMocapPipelineConfig(BaseModel):
@model_validator(mode="after")
def _build_tracker_config_from_detector_type(self) -> "PosthocMocapPipelineConfig":
if self.detector_type == "mediapipe":
- self.tracker_config = TrackerConfig(
- stages=[
- DetectionStageConfig(
- name="body",
- keypoint_detectors=[
- MediapipePoseDetectorConfig(
- model_complexity=self.mediapipe_model_complexity,
- num_poses=1,
- min_pose_detection_confidence=self.mediapipe_detection_confidence,
- min_pose_presence_confidence=self.mediapipe_presence_confidence,
- min_pose_tracking_confidence=self.mediapipe_tracking_confidence,
- ),
- MediapipeHandDetectorConfig(
- num_hands=self.mediapipe_num_hands,
- min_hand_detection_confidence=self.mediapipe_detection_confidence,
- min_hand_presence_confidence=self.mediapipe_presence_confidence,
- min_hand_tracking_confidence=self.mediapipe_tracking_confidence,
- ),
- MediapipeFaceDetectorConfig(
- num_faces=self.mediapipe_num_faces,
- min_face_detection_confidence=self.mediapipe_detection_confidence,
- min_face_presence_confidence=self.mediapipe_presence_confidence,
- min_face_tracking_confidence=self.mediapipe_tracking_confidence,
- ),
- ],
- )
- ]
+ self.tracker_config = build_mediapipe_tracker_config(
+ model_complexity=self.mediapipe_model_complexity,
+ detection_confidence=self.mediapipe_detection_confidence,
+ presence_confidence=self.mediapipe_presence_confidence,
+ tracking_confidence=self.mediapipe_tracking_confidence,
+ num_faces=self.mediapipe_num_faces,
)
else:
redetect_interval = max(1, round(5.0 * self.video_fps))
diff --git a/freemocap/core/tracking/tracker_factory.py b/freemocap/core/tracking/tracker_factory.py
index b446dc60..67b889c5 100644
--- a/freemocap/core/tracking/tracker_factory.py
+++ b/freemocap/core/tracking/tracker_factory.py
@@ -18,6 +18,8 @@ import skellytracker.core.detectors.object_detectors.yolox # noqa: F401 (
from skellytracker.core import (
DetectionStageConfig,
+ Keypoints,
+ Observation,
Tracker,
TrackerConfig,
)
@@ -28,6 +30,9 @@ from skellytracker.core.detectors.keypoint_detectors.charuco import (
from skellytracker.core.detectors.keypoint_detectors.rtmpose import (
RTMPoseDetectorConfig,
)
+from skellytracker.core.detectors.object_detectors.keypoint_bbox import (
+ KeypointBoundingBoxDetectorConfig,
+)
from skellytracker.core.detectors.object_detectors.yolox import (
YoloxPersonDetector,
YoloxPersonDetectorConfig,
@@ -143,19 +148,66 @@ def build_skeleton_tracker(
return Tracker.create(config, {"onnx": onnx_session})
-def build_mediapipe_tracker(
+def _mediapipe_hand_child_stage(
+ side: str,
+ *,
+ detection_confidence: float,
+ presence_confidence: float,
+ tracking_confidence: float,
+) -> DetectionStageConfig:
+ """Build a wrist-cropped hand child stage for the given side ('left'/'right').
+
+ Crops tightly around the wrist using the parent body stage's own
+ wrist/index/pinky keypoints (pure keypoint arithmetic, no inference β€” see
+ KeypointBoundingBoxDetectorConfig), then runs MediaPipe's hand landmarker
+ on that zoomed-in crop with a fixed handedness, since MediaPipe's own
+ per-crop handedness guess is unreliable on a tight single-hand crop.
+ """
+ from skellytracker.core.detectors.keypoint_detectors.mediapipe.hands.mediapipe_hand_detector import (
+ MediapipeHandDetectorConfig,
+ )
+
+ return DetectionStageConfig(
+ name=f"{side}_hand",
+ object_detector=KeypointBoundingBoxDetectorConfig(
+ center_keypoint_names=(f"{side}_index", f"{side}_pinky"),
+ scale_keypoint_pairs=(
+ (f"{side}_wrist", f"{side}_index"),
+ (f"{side}_wrist", f"{side}_pinky"),
+ ),
+ scale_factor=3.0,
+ min_box_size_px=120.0,
+ ),
+ keypoint_detectors=[
+ MediapipeHandDetectorConfig(
+ num_hands=1,
+ assumed_handedness=side,
+ min_hand_detection_confidence=detection_confidence,
+ min_hand_presence_confidence=presence_confidence,
+ min_hand_tracking_confidence=tracking_confidence,
+ )
+ ],
+ bbox_smoothing=BBoxSmoothingConfig(alpha=0.4),
+ )
+
+
+def build_mediapipe_tracker_config(
*,
model_complexity=None,
detection_confidence: float = 0.5,
presence_confidence: float = 0.5,
tracking_confidence: float = 0.5,
- num_hands: int = 2,
num_faces: int = 1,
-) -> tuple[Tracker, object]:
- """Build a MediaPipe body+hands+face Tracker backed by a MediaPipeSession.
+) -> TrackerConfig:
+ """Build a MediaPipe body+hands+face TrackerConfig.
- Returns (tracker, session). The session type is MediaPipeSession from
- skellytracker.core.sessions.mediapipe_session.
+ Pose is the parent "body" stage; left_hand/right_hand are wrist-cropped
+ child stages (see _mediapipe_hand_child_stage) instead of full-frame
+ sibling detectors, since hands occupy a tiny fraction of a full-body frame
+ and MediaPipe's palm detector struggles to find them at that scale. Face
+ is also a child stage (full-frame, no crop) purely to preserve the
+ existing pose/right_hand/left_hand/face point ordering that
+ merge_mediapipe_hand_face_children() and tracker_definitions.py expect.
"""
from skellytracker.core.detectors.keypoint_detectors.mediapipe.body.mediapipe_pose_detector import (
MediapipePoseDetectorConfig,
@@ -163,22 +215,14 @@ def build_mediapipe_tracker(
from skellytracker.core.detectors.keypoint_detectors.mediapipe.face.mediapipe_face_detector import (
MediapipeFaceDetectorConfig,
)
- from skellytracker.core.detectors.keypoint_detectors.mediapipe.hands.mediapipe_hand_detector import (
- MediapipeHandDetectorConfig,
- )
from skellytracker.core.detectors.keypoint_detectors.mediapipe.mediapipe_model_manager import (
MediapipePoseModelComplexity,
)
- from skellytracker.core.sessions.mediapipe_session import (
- MediaPipeSession,
- MediaPipeSessionConfig,
- )
if model_complexity is None:
model_complexity = MediapipePoseModelComplexity.HEAVY
- session = MediaPipeSession.create(MediaPipeSessionConfig())
- config = TrackerConfig(
+ return TrackerConfig(
stages=[
DetectionStageConfig(
name="body",
@@ -190,22 +234,92 @@ def build_mediapipe_tracker(
min_pose_presence_confidence=presence_confidence,
min_pose_tracking_confidence=tracking_confidence,
),
- MediapipeHandDetectorConfig(
- num_hands=num_hands,
- min_hand_detection_confidence=detection_confidence,
- min_hand_presence_confidence=presence_confidence,
- min_hand_tracking_confidence=tracking_confidence,
+ ],
+ children=[
+ _mediapipe_hand_child_stage(
+ "right",
+ detection_confidence=detection_confidence,
+ presence_confidence=presence_confidence,
+ tracking_confidence=tracking_confidence,
+ ),
+ _mediapipe_hand_child_stage(
+ "left",
+ detection_confidence=detection_confidence,
+ presence_confidence=presence_confidence,
+ tracking_confidence=tracking_confidence,
),
- MediapipeFaceDetectorConfig(
- num_faces=num_faces,
- min_face_detection_confidence=detection_confidence,
- min_face_presence_confidence=presence_confidence,
- min_face_tracking_confidence=tracking_confidence,
+ DetectionStageConfig(
+ name="face",
+ keypoint_detectors=[
+ MediapipeFaceDetectorConfig(
+ num_faces=num_faces,
+ min_face_detection_confidence=detection_confidence,
+ min_face_presence_confidence=presence_confidence,
+ min_face_tracking_confidence=tracking_confidence,
+ )
+ ],
),
],
keypoint_reset_policy=KeypointResetPolicyConfig(max_consecutive_misses=10),
)
]
)
- tracker = Tracker.create(config, {"mediapipe": session})
+
+
+def merge_mediapipe_hand_face_children(obs: Observation) -> None:
+ """Merge left_hand/right_hand/face child-stage keypoints back into the
+ top-level "body" stage, preserving the flat right_hand_*/left_hand_*
+ naming and pose+right_hand+left_hand+face ordering every downstream
+ consumer (skeleton_rigidifier, recording_status, export_to_blender,
+ tracker_definitions, freemocap-ui) expects. Mutates `obs` in place.
+ No-op for non-mediapipe configs (no "body" stage or no children).
+ """
+ body_stage = obs.stages.get("body")
+ if body_stage is None or not body_stage.children:
+ return
+ parts = [body_stage.keypoints] if body_stage.keypoints is not None else []
+ for name in ("right_hand", "left_hand", "face"):
+ child = body_stage.children.pop(name, None)
+ if child is not None and child.keypoints is not None:
+ parts.append(child.keypoints)
+ if len(parts) > 1:
+ body_stage.keypoints = Keypoints.concatenate(parts)
+
+
+def build_mediapipe_tracker(
+ *,
+ model_complexity=None,
+ detection_confidence: float = 0.5,
+ presence_confidence: float = 0.5,
+ tracking_confidence: float = 0.5,
+ num_hands: int = 2,
+ num_faces: int = 1,
+) -> tuple[Tracker, object]:
+ """Build a MediaPipe body+hands+face Tracker backed by a MediaPipeSession.
+
+ Returns (tracker, session). The session type is MediaPipeSession from
+ skellytracker.core.sessions.mediapipe_session.
+
+ `num_hands` is accepted for API compatibility but unused: each hand child
+ stage is a dedicated single-hand, wrist-cropped detector (see
+ _mediapipe_hand_child_stage), so MediaPipe always looks for exactly one
+ hand per stage rather than sharing a full-frame budget across both hands.
+ """
+ from skellytracker.core.sessions.mediapipe_session import (
+ MediaPipeSession,
+ MediaPipeSessionConfig,
+ )
+
+ session = MediaPipeSession.create(MediaPipeSessionConfig())
+ config = build_mediapipe_tracker_config(
+ model_complexity=model_complexity,
+ detection_confidence=detection_confidence,
+ presence_confidence=presence_confidence,
+ tracking_confidence=tracking_confidence,
+ num_faces=num_faces,
+ )
+ tracker = Tracker.create(
+ config,
+ {"mediapipe": session, "cpu": CpuSession.create(CpuSessionConfig())},
+ )
return tracker, session
diff --git a/uv.lock b/uv.lock
index 9c8da958..8c66da4b 100644
--- a/uv.lock
+++ b/uv.lock
@@ -491,7 +491,7 @@ wheels = [
[[package]]
name = "freemocap"
-version = "2.0.0a21"
+version = "2.0.0a23"
source = { editable = "." }
dependencies = [
{ name = "beartype" },
@@ -2165,7 +2165,7 @@ dependencies = [
[[package]]
name = "skellycam"
version = "2.0.0a6"
-source = { git = "https://github.com/freemocap/skellycam?branch=main#d091ecef176a83bb1bc648bdf5f2ee702e2612da" }
+source = { git = "https://github.com/freemocap/skellycam?branch=main#524e5fc513c99c72f20f76be0eccf7199b200dfb" }
dependencies = [
{ name = "av" },
{ name = "beartype" },
@@ -2223,7 +2223,7 @@ dependencies = [
[[package]]
name = "skellytracker"
version = "2024.9.1019"
-source = { git = "https://github.com/freemocap/skellytracker?branch=main#cb739aa5d059c90ba9c92ab5fbd987096423a638" }
+source = { git = "https://github.com/freemocap/skellytracker?branch=main#63100055867b161d32fb1535ed73f73e1ddd187d" }
dependencies = [
{ name = "beartype" },
{ name = "numpy" },