Free Motion Capture for Everyone πβ¨
TypeScript
Merging SfN25 changes into the development branch (#765)
jonmatthis48ffa50fd93e
diff --git a/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsx b/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsxindex 31d5a62e..f7650b28 100644--- a/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsx+++ b/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsx@@ -7,7 +7,11 @@ import {SphereGeometry,Object3D,Matrix4,- Vector3+ Vector3,+ BufferGeometry,+ BufferAttribute,+ LineBasicMaterial,+ LineSegments} from "three";import {extend, useFrame} from "@react-three/fiber";import {CameraControls, Grid} from "@react-three/drei";@@ -25,9 +29,39 @@ interface Point3d {z: number;}+// Segment definitions with virtual points that need to be calculated+const SEGMENT_DEFINITIONS: Record<string, {proximal: string, distal: string}> = {+ head: { proximal: "body.left_ear", distal: "body.right_ear" },+ neck: { proximal: "head_center", distal: "neck_center" },+ spine: { proximal: "neck_center", distal: "hips_center" },+ right_shoulder: { proximal: "neck_center", distal: "body.right_shoulder" },+ left_shoulder: { proximal: "neck_center", distal: "body.left_shoulder" },+ right_upper_arm: { proximal: "body.right_shoulder", distal: "body.right_elbow" },+ left_upper_arm: { proximal: "body.left_shoulder", distal: "body.left_elbow" },+ right_forearm: { proximal: "body.right_elbow", distal: "body.right_wrist" },+ left_forearm: { proximal: "body.left_elbow", distal: "body.left_wrist" },+ right_hand: { proximal: "body.right_wrist", distal: "body.right_index" },+ left_hand: { proximal: "body.left_wrist", distal: "body.left_index" },+ right_pelvis: { proximal: "hips_center", distal: "body.right_hip" },+ left_pelvis: { proximal: "hips_center", distal: "body.left_hip" },+ right_thigh: { proximal: "body.right_hip", distal: "body.right_knee" },+ left_thigh: { proximal: "body.left_hip", distal: "body.left_knee" },+ right_shank: { proximal: "body.right_knee", distal: "body.right_ankle" },+ left_shank: { proximal: "body.left_knee", distal: "body.left_ankle" },+ right_foot: { proximal: "body.right_ankle", distal: "body.right_foot_index" },+ left_foot: { proximal: "body.left_ankle", distal: "body.left_foot_index" },+ right_heel: { proximal: "body.right_ankle", distal: "body.right_heel" },+ left_heel: { proximal: "body.left_ankle", distal: "body.left_heel" },+ right_foot_bottom: { proximal: "body.right_heel", distal: "body.right_foot_index" },+ left_foot_bottom: { proximal: "body.left_heel", distal: "body.left_foot_index" },+};++const MAX_SEGMENTS = Object.keys(SEGMENT_DEFINITIONS).length;+export function ThreeJsScene() {const { connectedCameraIds, subscribeToTrackedPoints } = useServer();const instancedMeshRef = useRef<InstancedMesh>(null);+ const lineSegmentsRef = useRef<LineSegments>(null);const cameraRefs = useRef<PerspectiveCamera[]>([]);// Performance-critical: Direct refs to avoid React re-renders@@ -35,17 +69,39 @@ export function ThreeJsScene() {const pointNameToIndexRef = useRef<Map<string, number>>(new Map());const nextAvailableIndexRef = useRef<number>(0);const dirtyIndicesRef = useRef<Set<number>>(new Set());+ const needsLineUpdate = useRef<boolean>(false);// Pre-create geometry and material once- const sphereGeometry = useMemo(() => new SphereGeometry(0.05, 8, 6), []); // Bigger spheres+ const sphereGeometry = useMemo(() => new SphereGeometry(0.4, 8, 6), []); // Bigger spheresconst sphereMaterial = useMemo(() => new MeshStandardMaterial({- color: '#ff0066', // Brighter color for visibility+ color: '#00ff66', // Brighter color for visibilityroughness: 0.4,metalness: 0.6,- emissive: '#ff0066',+ emissive: '#00ff66',emissiveIntensity: 0.2}), []);+ // Create line material+ const lineMaterial = useMemo(() => new LineBasicMaterial({+ color: '#ffffff',+ linewidth: 2,+ transparent: true,+ opacity: 0.8+ }), []);++ // Create line geometry with pre-allocated buffer+ const lineGeometry = useMemo(() => {+ const geometry = new BufferGeometry();+ // Each segment needs 2 vertices (6 floats)+ const positions = new Float32Array(MAX_SEGMENTS * 2 * 3);+ // Initialize far away+ for (let i = 0; i < positions.length; i++) {+ positions[i] = 10000;+ }+ geometry.setAttribute('position', new BufferAttribute(positions, 3));+ return geometry;+ }, []);+// Calculate positions for cameras in a gridconst cameraPositions = useMemo(() => {const numCameras = connectedCameraIds.length;@@ -68,12 +124,51 @@ export function ThreeJsScene() {});}, [connectedCameraIds]);+ // Helper function to calculate virtual points+ const calculateVirtualPoint = (points: Map<string, Point3d>, virtualName: string): Point3d | null => {+ switch(virtualName) {+ case "head_center": {+ const leftEar = points.get("body.left_ear");+ const rightEar = points.get("body.right_ear");+ if (!leftEar || !rightEar) return null;+ return {+ x: (leftEar.x + rightEar.x) / 2,+ y: (leftEar.y + rightEar.y) / 2,+ z: (leftEar.z + rightEar.z) / 2+ };+ }+ case "neck_center": {+ const leftShoulder = points.get("body.left_shoulder");+ const rightShoulder = points.get("body.right_shoulder");+ if (!leftShoulder || !rightShoulder) return null;+ return {+ x: (leftShoulder.x + rightShoulder.x) / 2,+ y: (leftShoulder.y + rightShoulder.y) / 2,+ z: (leftShoulder.z + rightShoulder.z) / 2+ };+ }+ case "hips_center": {+ const leftHip = points.get("body.left_hip");+ const rightHip = points.get("body.right_hip");+ if (!leftHip || !rightHip) return null;+ return {+ x: (leftHip.x + rightHip.x) / 2,+ y: (leftHip.y + rightHip.y) / 2,+ z: (leftHip.z + rightHip.z) / 2+ };+ }+ default:+ return points.get(virtualName) || null;+ }+ };+// Subscribe to tracked points updatesuseEffect(() => {const unsubscribe = subscribeToTrackedPoints((newPoints: Map<string, Point3d>) => {// Only update the ref and mark dirty indices - no React state updates!const prevPoints = trackedPointsRef.current;trackedPointsRef.current = newPoints;+ needsLineUpdate.current = true; // Mark lines for update// Find new points that need indices assignedfor (const [pointName, point] of newPoints) {@@ -83,7 +178,6 @@ export function ThreeJsScene() {console.error(`Maximum points (${MAX_POINTS}) exceeded!`);continue;}-const newIndex = nextAvailableIndexRef.current++;pointNameToIndexRef.current.set(pointName, newIndex);dirtyIndicesRef.current.add(newIndex);@@ -139,50 +233,84 @@ export function ThreeJsScene() {}});- // Only update if we have dirty indices- if (!instancedMeshRef.current || dirtyIndicesRef.current.size === 0) return;-- const mesh = instancedMeshRef.current;- const points = trackedPointsRef.current;- const nameToIndex = pointNameToIndexRef.current;+ const scale = 1 // Adjust this to fit your sceneconst farAway = new Vector3(10000, 10000, 10000);- // Process only dirty indices- for (const index of dirtyIndicesRef.current) {- // Find the point name for this index- let pointName: string | null = null;- for (const [name, idx] of nameToIndex) {- if (idx === index) {- pointName = name;- break;+ // Update points if we have dirty indices+ if (instancedMeshRef.current && dirtyIndicesRef.current.size > 0) {+ const mesh = instancedMeshRef.current;+ const points = trackedPointsRef.current;+ const nameToIndex = pointNameToIndexRef.current;++ // Process only dirty indices+ for (const index of dirtyIndicesRef.current) {+ // Find the point name for this index+ let pointName: string | null = null;+ for (const [name, idx] of nameToIndex) {+ if (idx === index) {+ pointName = name;+ break;+ }+ }++ if (pointName && points.has(pointName)) {+ // Update position for visible point+ const point = points.get(pointName)!;+ DUMMY.position.set(+ point.x * scale,+ point.y * scale,+ (point.z - 15) * scale // Center around z=15 then scale+ );+ DUMMY.scale.set(POINT_SCALE, POINT_SCALE, POINT_SCALE);+ } else {+ // Hide removed/missing points+ DUMMY.position.copy(farAway);+ DUMMY.scale.set(0, 0, 0);}- }- if (pointName && points.has(pointName)) {- // Update position for visible point- const point = points.get(pointName)!;- // Scale coordinates to bring them closer to origin- const scale = 0.2; // Adjust this to fit your scene- DUMMY.position.set(- point.x * scale,- point.y * scale,- (point.z - 15) * scale // Center around z=15 then scale- );- DUMMY.scale.set(POINT_SCALE, POINT_SCALE, POINT_SCALE);- } else {- // Hide removed/missing points- DUMMY.position.copy(farAway);- DUMMY.scale.set(0, 0, 0);+ DUMMY.updateMatrix();+ mesh.setMatrixAt(index, DUMMY.matrix);}- DUMMY.updateMatrix();- mesh.setMatrixAt(index, DUMMY.matrix);+ // Only update the GPU buffer if we made changes+ if (dirtyIndicesRef.current.size > 0) {+ mesh.instanceMatrix.needsUpdate = true;+ dirtyIndicesRef.current.clear();+ }}- // Only update the GPU buffer if we made changes- if (dirtyIndicesRef.current.size > 0) {- mesh.instanceMatrix.needsUpdate = true;- dirtyIndicesRef.current.clear();+ // Update line segments+ if (lineSegmentsRef.current && needsLineUpdate.current) {+ const points = trackedPointsRef.current;+ const positions = lineGeometry.attributes.position.array as Float32Array;+ let segmentIndex = 0;++ for (const [segmentName, segment] of Object.entries(SEGMENT_DEFINITIONS)) {+ const proximalPoint = calculateVirtualPoint(points, segment.proximal);+ const distalPoint = calculateVirtualPoint(points, segment.distal);++ if (proximalPoint && distalPoint) {+ // Set proximal vertex+ positions[segmentIndex * 6] = proximalPoint.x * scale;+ positions[segmentIndex * 6 + 1] = proximalPoint.y * scale;+ positions[segmentIndex * 6 + 2] = (proximalPoint.z - 15) * scale;++ // Set distal vertex+ positions[segmentIndex * 6 + 3] = distalPoint.x * scale;+ positions[segmentIndex * 6 + 4] = distalPoint.y * scale;+ positions[segmentIndex * 6 + 5] = (distalPoint.z - 15) * scale;+ } else {+ // Hide segment if points not available+ for (let i = 0; i < 6; i++) {+ positions[segmentIndex * 6 + i] = 10000;+ }+ }++ segmentIndex++;+ }++ lineGeometry.attributes.position.needsUpdate = true;+ needsLineUpdate.current = false;}});@@ -190,10 +318,6 @@ export function ThreeJsScene() {<><CameraControls makeDefault/>- {cameraRefs.current.map((camera, index) => camera && (- <cameraHelper key={index} args={[camera]} />- ))}-<ambientLight intensity={0.5}/><directionalLightcastShadow@@ -211,7 +335,7 @@ export function ThreeJsScene() {sectionSize={3}sectionThickness={1}//@ts-ignore- sectionColor={[0.5, 0, 0.5]}+ sectionColor={[0.25, 0, 0.25]}fadeDistance={100}/>@@ -223,6 +347,14 @@ export function ThreeJsScene() {args={[sphereGeometry, sphereMaterial, MAX_POINTS]}frustumCulled={false}/>++ {/* Line segments for body connections */}+ <lineSegments+ ref={lineSegmentsRef}+ geometry={lineGeometry}+ material={lineMaterial}+ frustumCulled={false}+ /></>);-}+}\ No newline at end of filediff --git a/freemocap/api/http/mocap/mocap_router.py b/freemocap/api/http/mocap/mocap_router.pyindex 4c96a3dc..de53c56e 100644--- a/freemocap/api/http/mocap/mocap_router.py+++ b/freemocap/api/http/mocap/mocap_router.py@@ -32,7 +32,7 @@ class StartMocapRecordingRequest(BaseModel):mocap_task_config: MocapPipelineTaskConfig = Field(alias="mocapTaskConfig")def to_recording_info(self) -> RecordingInfo:- recordings_directory = Path(self.mocap_recording_directory).parent+ recordings_directory = Path(self.mocap_recording_directory).expanduser().parentrecording_name = Path(self.mocap_recording_directory).stemif not recording_name.endswith('_mocap'):recording_name += '_mocap'@@ -43,6 +43,7 @@ class StartMocapRecordingRequest(BaseModel):)+class StopMocapRecordingRequest(BaseModel):mocap_task_config: MocapPipelineTaskConfig = Field(alias="mocapTaskConfig")diff --git a/freemocap/core/pipeline/posthoc_pipelines/posthoc_calibration_pipeline/calibration_helpers/charuco_observation_aggregator.py b/freemocap/core/pipeline/posthoc_pipelines/posthoc_calibration_pipeline/calibration_helpers/charuco_observation_aggregator.pyindex 6b93e209..8ad97cb4 100644--- a/freemocap/core/pipeline/posthoc_pipelines/posthoc_calibration_pipeline/calibration_helpers/charuco_observation_aggregator.py+++ b/freemocap/core/pipeline/posthoc_pipelines/posthoc_calibration_pipeline/calibration_helpers/charuco_observation_aggregator.py@@ -94,7 +94,7 @@ def anipose_calibration_from_charuco_observations(video_metadata: dict[VideoIdString, VideoMetadata],recording_info: RecordingInfo,pin_camera_0_to_origin: bool = True,- use_charuco_as_groundplane: bool = True,+ use_charuco_as_groundplane: bool = False,init_intrinsics: bool = True,init_extrinsics: bool = True,verbose: bool = True, ) -> Path:diff --git a/freemocap/core/pipeline/posthoc_pipelines/posthoc_mocap_pipeline/mocap_helpers/triangulate_trajectory_array.py b/freemocap/core/pipeline/posthoc_pipelines/posthoc_mocap_pipeline/mocap_helpers/triangulate_trajectory_array.pyindex e2c5e94f..27dce683 100644--- a/freemocap/core/pipeline/posthoc_pipelines/posthoc_mocap_pipeline/mocap_helpers/triangulate_trajectory_array.py+++ b/freemocap/core/pipeline/posthoc_pipelines/posthoc_mocap_pipeline/mocap_helpers/triangulate_trajectory_array.py@@ -108,14 +108,23 @@ def triangulate_frame_observations(frame_number: int,config=config,calculate_reprojection_error=calculate_reprojection_error,)++ rotated_triangulated = rotate_by_180_deg_about_x(triangulated_data)+return Observation3d(frame_number=frame_number,- triangulated_data=np.squeeze(triangulated_data),+ triangulated_data=np.squeeze(rotated_triangulated),names=list(frame_observations_by_camera.values())[0].to_tracked_points().keys(),# reprojection_error=reprojection_error,# reprojection_error_by_camera=reprojection_error_by_camera,)+def rotate_by_180_deg_about_x(points_3d: np.ndarray) -> np.ndarray:+ rotation_matrix = np.array([[1, 0, 0],+ [0, -1, 0],+ [0, 0, -1]])+ rotated_points = points_3d @ rotation_matrix.T+ return rotated_pointsdef triangulate_frame_groups(frame_groups: dict[int, FrameObservationsByCamera],