Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

Merging SfN25 changes into the development branch (#765)

jonmatthis48ffa50fd93e
diff --git a/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsx b/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsx
index 31d5a62e..f7650b28 100644
--- a/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsx
+++ b/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsx
@@ -7,7 +7,11 @@ import {
SphereGeometry,
Object3D,
Matrix4,
- Vector3
+ Vector3,
+ BufferGeometry,
+ BufferAttribute,
+ LineBasicMaterial,
+ LineSegments
} from "three";
import {extend, useFrame} from "@react-three/fiber";
import {CameraControls, Grid} from "@react-three/drei";
@@ -25,9 +29,39 @@ interface Point3d {
z: number;
}
+// Segment definitions with virtual points that need to be calculated
+const SEGMENT_DEFINITIONS: Record<string, {proximal: string, distal: string}> = {
+ head: { proximal: "body.left_ear", distal: "body.right_ear" },
+ neck: { proximal: "head_center", distal: "neck_center" },
+ spine: { proximal: "neck_center", distal: "hips_center" },
+ right_shoulder: { proximal: "neck_center", distal: "body.right_shoulder" },
+ left_shoulder: { proximal: "neck_center", distal: "body.left_shoulder" },
+ right_upper_arm: { proximal: "body.right_shoulder", distal: "body.right_elbow" },
+ left_upper_arm: { proximal: "body.left_shoulder", distal: "body.left_elbow" },
+ right_forearm: { proximal: "body.right_elbow", distal: "body.right_wrist" },
+ left_forearm: { proximal: "body.left_elbow", distal: "body.left_wrist" },
+ right_hand: { proximal: "body.right_wrist", distal: "body.right_index" },
+ left_hand: { proximal: "body.left_wrist", distal: "body.left_index" },
+ right_pelvis: { proximal: "hips_center", distal: "body.right_hip" },
+ left_pelvis: { proximal: "hips_center", distal: "body.left_hip" },
+ right_thigh: { proximal: "body.right_hip", distal: "body.right_knee" },
+ left_thigh: { proximal: "body.left_hip", distal: "body.left_knee" },
+ right_shank: { proximal: "body.right_knee", distal: "body.right_ankle" },
+ left_shank: { proximal: "body.left_knee", distal: "body.left_ankle" },
+ right_foot: { proximal: "body.right_ankle", distal: "body.right_foot_index" },
+ left_foot: { proximal: "body.left_ankle", distal: "body.left_foot_index" },
+ right_heel: { proximal: "body.right_ankle", distal: "body.right_heel" },
+ left_heel: { proximal: "body.left_ankle", distal: "body.left_heel" },
+ right_foot_bottom: { proximal: "body.right_heel", distal: "body.right_foot_index" },
+ left_foot_bottom: { proximal: "body.left_heel", distal: "body.left_foot_index" },
+};
+
+const MAX_SEGMENTS = Object.keys(SEGMENT_DEFINITIONS).length;
+
export function ThreeJsScene() {
const { connectedCameraIds, subscribeToTrackedPoints } = useServer();
const instancedMeshRef = useRef<InstancedMesh>(null);
+ const lineSegmentsRef = useRef<LineSegments>(null);
const cameraRefs = useRef<PerspectiveCamera[]>([]);
// Performance-critical: Direct refs to avoid React re-renders
@@ -35,17 +69,39 @@ export function ThreeJsScene() {
const pointNameToIndexRef = useRef<Map<string, number>>(new Map());
const nextAvailableIndexRef = useRef<number>(0);
const dirtyIndicesRef = useRef<Set<number>>(new Set());
+ const needsLineUpdate = useRef<boolean>(false);
// Pre-create geometry and material once
- const sphereGeometry = useMemo(() => new SphereGeometry(0.05, 8, 6), []); // Bigger spheres
+ const sphereGeometry = useMemo(() => new SphereGeometry(0.4, 8, 6), []); // Bigger spheres
const sphereMaterial = useMemo(() => new MeshStandardMaterial({
- color: '#ff0066', // Brighter color for visibility
+ color: '#00ff66', // Brighter color for visibility
roughness: 0.4,
metalness: 0.6,
- emissive: '#ff0066',
+ emissive: '#00ff66',
emissiveIntensity: 0.2
}), []);
+ // Create line material
+ const lineMaterial = useMemo(() => new LineBasicMaterial({
+ color: '#ffffff',
+ linewidth: 2,
+ transparent: true,
+ opacity: 0.8
+ }), []);
+
+ // Create line geometry with pre-allocated buffer
+ const lineGeometry = useMemo(() => {
+ const geometry = new BufferGeometry();
+ // Each segment needs 2 vertices (6 floats)
+ const positions = new Float32Array(MAX_SEGMENTS * 2 * 3);
+ // Initialize far away
+ for (let i = 0; i < positions.length; i++) {
+ positions[i] = 10000;
+ }
+ geometry.setAttribute('position', new BufferAttribute(positions, 3));
+ return geometry;
+ }, []);
+
// Calculate positions for cameras in a grid
const cameraPositions = useMemo(() => {
const numCameras = connectedCameraIds.length;
@@ -68,12 +124,51 @@ export function ThreeJsScene() {
});
}, [connectedCameraIds]);
+ // Helper function to calculate virtual points
+ const calculateVirtualPoint = (points: Map<string, Point3d>, virtualName: string): Point3d | null => {
+ switch(virtualName) {
+ case "head_center": {
+ const leftEar = points.get("body.left_ear");
+ const rightEar = points.get("body.right_ear");
+ if (!leftEar || !rightEar) return null;
+ return {
+ x: (leftEar.x + rightEar.x) / 2,
+ y: (leftEar.y + rightEar.y) / 2,
+ z: (leftEar.z + rightEar.z) / 2
+ };
+ }
+ case "neck_center": {
+ const leftShoulder = points.get("body.left_shoulder");
+ const rightShoulder = points.get("body.right_shoulder");
+ if (!leftShoulder || !rightShoulder) return null;
+ return {
+ x: (leftShoulder.x + rightShoulder.x) / 2,
+ y: (leftShoulder.y + rightShoulder.y) / 2,
+ z: (leftShoulder.z + rightShoulder.z) / 2
+ };
+ }
+ case "hips_center": {
+ const leftHip = points.get("body.left_hip");
+ const rightHip = points.get("body.right_hip");
+ if (!leftHip || !rightHip) return null;
+ return {
+ x: (leftHip.x + rightHip.x) / 2,
+ y: (leftHip.y + rightHip.y) / 2,
+ z: (leftHip.z + rightHip.z) / 2
+ };
+ }
+ default:
+ return points.get(virtualName) || null;
+ }
+ };
+
// Subscribe to tracked points updates
useEffect(() => {
const unsubscribe = subscribeToTrackedPoints((newPoints: Map<string, Point3d>) => {
// Only update the ref and mark dirty indices - no React state updates!
const prevPoints = trackedPointsRef.current;
trackedPointsRef.current = newPoints;
+ needsLineUpdate.current = true; // Mark lines for update
// Find new points that need indices assigned
for (const [pointName, point] of newPoints) {
@@ -83,7 +178,6 @@ export function ThreeJsScene() {
console.error(`Maximum points (${MAX_POINTS}) exceeded!`);
continue;
}
-
const newIndex = nextAvailableIndexRef.current++;
pointNameToIndexRef.current.set(pointName, newIndex);
dirtyIndicesRef.current.add(newIndex);
@@ -139,50 +233,84 @@ export function ThreeJsScene() {
}
});
- // Only update if we have dirty indices
- if (!instancedMeshRef.current || dirtyIndicesRef.current.size === 0) return;
-
- const mesh = instancedMeshRef.current;
- const points = trackedPointsRef.current;
- const nameToIndex = pointNameToIndexRef.current;
+ const scale = 1 // Adjust this to fit your scene
const farAway = new Vector3(10000, 10000, 10000);
- // Process only dirty indices
- for (const index of dirtyIndicesRef.current) {
- // Find the point name for this index
- let pointName: string | null = null;
- for (const [name, idx] of nameToIndex) {
- if (idx === index) {
- pointName = name;
- break;
+ // Update points if we have dirty indices
+ if (instancedMeshRef.current && dirtyIndicesRef.current.size > 0) {
+ const mesh = instancedMeshRef.current;
+ const points = trackedPointsRef.current;
+ const nameToIndex = pointNameToIndexRef.current;
+
+ // Process only dirty indices
+ for (const index of dirtyIndicesRef.current) {
+ // Find the point name for this index
+ let pointName: string | null = null;
+ for (const [name, idx] of nameToIndex) {
+ if (idx === index) {
+ pointName = name;
+ break;
+ }
+ }
+
+ if (pointName && points.has(pointName)) {
+ // Update position for visible point
+ const point = points.get(pointName)!;
+ DUMMY.position.set(
+ point.x * scale,
+ point.y * scale,
+ (point.z - 15) * scale // Center around z=15 then scale
+ );
+ DUMMY.scale.set(POINT_SCALE, POINT_SCALE, POINT_SCALE);
+ } else {
+ // Hide removed/missing points
+ DUMMY.position.copy(farAway);
+ DUMMY.scale.set(0, 0, 0);
}
- }
- if (pointName && points.has(pointName)) {
- // Update position for visible point
- const point = points.get(pointName)!;
- // Scale coordinates to bring them closer to origin
- const scale = 0.2; // Adjust this to fit your scene
- DUMMY.position.set(
- point.x * scale,
- point.y * scale,
- (point.z - 15) * scale // Center around z=15 then scale
- );
- DUMMY.scale.set(POINT_SCALE, POINT_SCALE, POINT_SCALE);
- } else {
- // Hide removed/missing points
- DUMMY.position.copy(farAway);
- DUMMY.scale.set(0, 0, 0);
+ DUMMY.updateMatrix();
+ mesh.setMatrixAt(index, DUMMY.matrix);
}
- DUMMY.updateMatrix();
- mesh.setMatrixAt(index, DUMMY.matrix);
+ // Only update the GPU buffer if we made changes
+ if (dirtyIndicesRef.current.size > 0) {
+ mesh.instanceMatrix.needsUpdate = true;
+ dirtyIndicesRef.current.clear();
+ }
}
- // Only update the GPU buffer if we made changes
- if (dirtyIndicesRef.current.size > 0) {
- mesh.instanceMatrix.needsUpdate = true;
- dirtyIndicesRef.current.clear();
+ // Update line segments
+ if (lineSegmentsRef.current && needsLineUpdate.current) {
+ const points = trackedPointsRef.current;
+ const positions = lineGeometry.attributes.position.array as Float32Array;
+ let segmentIndex = 0;
+
+ for (const [segmentName, segment] of Object.entries(SEGMENT_DEFINITIONS)) {
+ const proximalPoint = calculateVirtualPoint(points, segment.proximal);
+ const distalPoint = calculateVirtualPoint(points, segment.distal);
+
+ if (proximalPoint && distalPoint) {
+ // Set proximal vertex
+ positions[segmentIndex * 6] = proximalPoint.x * scale;
+ positions[segmentIndex * 6 + 1] = proximalPoint.y * scale;
+ positions[segmentIndex * 6 + 2] = (proximalPoint.z - 15) * scale;
+
+ // Set distal vertex
+ positions[segmentIndex * 6 + 3] = distalPoint.x * scale;
+ positions[segmentIndex * 6 + 4] = distalPoint.y * scale;
+ positions[segmentIndex * 6 + 5] = (distalPoint.z - 15) * scale;
+ } else {
+ // Hide segment if points not available
+ for (let i = 0; i < 6; i++) {
+ positions[segmentIndex * 6 + i] = 10000;
+ }
+ }
+
+ segmentIndex++;
+ }
+
+ lineGeometry.attributes.position.needsUpdate = true;
+ needsLineUpdate.current = false;
}
});
@@ -190,10 +318,6 @@ export function ThreeJsScene() {
<>
<CameraControls makeDefault/>
- {cameraRefs.current.map((camera, index) => camera && (
- <cameraHelper key={index} args={[camera]} />
- ))}
-
<ambientLight intensity={0.5}/>
<directionalLight
castShadow
@@ -211,7 +335,7 @@ export function ThreeJsScene() {
sectionSize={3}
sectionThickness={1}
//@ts-ignore
- sectionColor={[0.5, 0, 0.5]}
+ sectionColor={[0.25, 0, 0.25]}
fadeDistance={100}
/>
@@ -223,6 +347,14 @@ export function ThreeJsScene() {
args={[sphereGeometry, sphereMaterial, MAX_POINTS]}
frustumCulled={false}
/>
+
+ {/* Line segments for body connections */}
+ <lineSegments
+ ref={lineSegmentsRef}
+ geometry={lineGeometry}
+ material={lineMaterial}
+ frustumCulled={false}
+ />
</>
);
-}
+}
\ No newline at end of file
diff --git a/freemocap/api/http/mocap/mocap_router.py b/freemocap/api/http/mocap/mocap_router.py
index 4c96a3dc..de53c56e 100644
--- a/freemocap/api/http/mocap/mocap_router.py
+++ b/freemocap/api/http/mocap/mocap_router.py
@@ -32,7 +32,7 @@ class StartMocapRecordingRequest(BaseModel):
mocap_task_config: MocapPipelineTaskConfig = Field(alias="mocapTaskConfig")
def to_recording_info(self) -> RecordingInfo:
- recordings_directory = Path(self.mocap_recording_directory).parent
+ recordings_directory = Path(self.mocap_recording_directory).expanduser().parent
recording_name = Path(self.mocap_recording_directory).stem
if not recording_name.endswith('_mocap'):
recording_name += '_mocap'
@@ -43,6 +43,7 @@ class StartMocapRecordingRequest(BaseModel):
)
+
class StopMocapRecordingRequest(BaseModel):
mocap_task_config: MocapPipelineTaskConfig = Field(alias="mocapTaskConfig")
diff --git a/freemocap/core/pipeline/posthoc_pipelines/posthoc_calibration_pipeline/calibration_helpers/charuco_observation_aggregator.py b/freemocap/core/pipeline/posthoc_pipelines/posthoc_calibration_pipeline/calibration_helpers/charuco_observation_aggregator.py
index 6b93e209..8ad97cb4 100644
--- a/freemocap/core/pipeline/posthoc_pipelines/posthoc_calibration_pipeline/calibration_helpers/charuco_observation_aggregator.py
+++ b/freemocap/core/pipeline/posthoc_pipelines/posthoc_calibration_pipeline/calibration_helpers/charuco_observation_aggregator.py
@@ -94,7 +94,7 @@ def anipose_calibration_from_charuco_observations(
video_metadata: dict[VideoIdString, VideoMetadata],
recording_info: RecordingInfo,
pin_camera_0_to_origin: bool = True,
- use_charuco_as_groundplane: bool = True,
+ use_charuco_as_groundplane: bool = False,
init_intrinsics: bool = True,
init_extrinsics: bool = True,
verbose: bool = True, ) -> Path:
diff --git a/freemocap/core/pipeline/posthoc_pipelines/posthoc_mocap_pipeline/mocap_helpers/triangulate_trajectory_array.py b/freemocap/core/pipeline/posthoc_pipelines/posthoc_mocap_pipeline/mocap_helpers/triangulate_trajectory_array.py
index e2c5e94f..27dce683 100644
--- a/freemocap/core/pipeline/posthoc_pipelines/posthoc_mocap_pipeline/mocap_helpers/triangulate_trajectory_array.py
+++ b/freemocap/core/pipeline/posthoc_pipelines/posthoc_mocap_pipeline/mocap_helpers/triangulate_trajectory_array.py
@@ -108,14 +108,23 @@ def triangulate_frame_observations(frame_number: int,
config=config,
calculate_reprojection_error=calculate_reprojection_error,
)
+
+ rotated_triangulated = rotate_by_180_deg_about_x(triangulated_data)
+
return Observation3d(
frame_number=frame_number,
- triangulated_data=np.squeeze(triangulated_data),
+ triangulated_data=np.squeeze(rotated_triangulated),
names=list(frame_observations_by_camera.values())[0].to_tracked_points().keys(),
# reprojection_error=reprojection_error,
# reprojection_error_by_camera=reprojection_error_by_camera,
)
+def rotate_by_180_deg_about_x(points_3d: np.ndarray) -> np.ndarray:
+ rotation_matrix = np.array([[1, 0, 0],
+ [0, -1, 0],
+ [0, 0, -1]])
+ rotated_points = points_3d @ rotation_matrix.T
+ return rotated_points
def triangulate_frame_groups(
frame_groups: dict[int, FrameObservationsByCamera],