Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

gettin there

jonmatthiscb148bb82842
diff --git a/freemocap-ui/src/components/viewport3d/RigidBodyRenderer.tsx b/freemocap-ui/src/components/old-viewport3d/RigidBodyRenderer.tsx
similarity index 98%
rename from freemocap-ui/src/components/viewport3d/RigidBodyRenderer.tsx
rename to freemocap-ui/src/components/old-viewport3d/RigidBodyRenderer.tsx
index d5f4a99f..bcea9bf6 100644
--- a/freemocap-ui/src/components/viewport3d/RigidBodyRenderer.tsx
+++ b/freemocap-ui/src/components/old-viewport3d/RigidBodyRenderer.tsx
@@ -2,7 +2,7 @@ import {useEffect, useMemo, useRef} from "react";
import {Color, CylinderGeometry, InstancedMesh, MeshStandardMaterial, Object3D, Vector3,} from "three";
import {useFrame} from "@react-three/fiber";
import {useServer} from "@/services";
-import {RigidBodyPose} from "@/components/viewport3d/viewport3d-types";
+import {RigidBodyPose} from "@/components/viewport3d/helpers/viewport3d-types";
import {Z_OFFSET} from "@/components/viewport3d/skeleton-config";
import {SKELETON_COLORS} from "@/components/viewport3d/skeleton-colors";
diff --git a/freemocap-ui/src/components/viewport3d/SceneEnvironment.tsx b/freemocap-ui/src/components/old-viewport3d/SceneEnvironment.tsx
similarity index 100%
rename from freemocap-ui/src/components/viewport3d/SceneEnvironment.tsx
rename to freemocap-ui/src/components/old-viewport3d/SceneEnvironment.tsx
diff --git a/freemocap-ui/src/components/viewport3d/SkeletonRenderer.tsx b/freemocap-ui/src/components/old-viewport3d/SkeletonRenderer.tsx
similarity index 99%
rename from freemocap-ui/src/components/viewport3d/SkeletonRenderer.tsx
rename to freemocap-ui/src/components/old-viewport3d/SkeletonRenderer.tsx
index 59257a26..f7270691 100644
--- a/freemocap-ui/src/components/viewport3d/SkeletonRenderer.tsx
+++ b/freemocap-ui/src/components/old-viewport3d/SkeletonRenderer.tsx
@@ -12,7 +12,7 @@ import {
} from "three";
import {useFrame} from "@react-three/fiber";
import {useServer} from "@/services";
-import {Point3d, PointStyle} from "@/components/viewport3d/viewport3d-types";
+import {Point3d, PointStyle} from "@/components/viewport3d/helpers/viewport3d-types";
import {
getPointStyle,
MAX_POINTS,
diff --git a/freemocap-ui/src/components/old-viewport3d/ThreeJsCanvas.tsx b/freemocap-ui/src/components/old-viewport3d/ThreeJsCanvas.tsx
new file mode 100644
index 00000000..fc10ae91
--- /dev/null
+++ b/freemocap-ui/src/components/old-viewport3d/ThreeJsCanvas.tsx
@@ -0,0 +1,100 @@
+import {useCallback, useEffect, useRef} from "react";
+import {Canvas} from "@react-three/fiber";
+import {Box, IconButton, Tooltip, Typography} from "@mui/material";
+import CenterFocusStrongIcon from "@mui/icons-material/CenterFocusStrong";
+import HomeIcon from "@mui/icons-material/Home";
+import type CameraControlsImpl from "camera-controls";
+import {ThreeJsScene} from "@/components/viewport3d/ThreeJsScene";
+import {fitCameraToSkeleton} from "@/components/viewport3d/fit-camera";
+import {useServer} from "@/services";
+
+export function ThreeJsCanvas() {
+ const cameraControlsRef = useRef<CameraControlsImpl>(null!);
+ const containerRef = useRef<HTMLDivElement>(null);
+ const { getLatestTrackedPoints } = useServer();
+
+ const handleFitToSkeleton = useCallback(() => {
+ fitCameraToSkeleton(cameraControlsRef.current, getLatestTrackedPoints());
+ }, [getLatestTrackedPoints]);
+
+ // Press "F" while the viewport container is focused to fit camera to skeleton
+ useEffect(() => {
+ const container = containerRef.current;
+ if (!container) return;
+
+ const handleKeyDown = (e: KeyboardEvent) => {
+ if (e.key === "f" || e.key === "F") {
+ e.preventDefault();
+ handleFitToSkeleton();
+ }
+ };
+
+ container.addEventListener("keydown", handleKeyDown);
+ return () => container.removeEventListener("keydown", handleKeyDown);
+ }, [handleFitToSkeleton]);
+
+ return (
+ <Box
+ ref={containerRef}
+ tabIndex={0}
+ sx={{
+ width: '100%',
+ height: '100%',
+ position: 'relative',
+ outline: 'none',
+ }}
+ >
+ <Canvas
+ shadows
+ camera={{ position: [1e3, 1e3, 1e3], fov: 75, near: 0.1, far: 10000 }}
+ gl={{ antialias: true, logarithmicDepthBuffer: true }}
+ >
+ <ThreeJsScene cameraControlsRef={cameraControlsRef} />
+ </Canvas>
+
+ <Box sx={{ position: 'absolute', left: 16, bottom: 16, bgcolor: 'background.paper', p: 1, borderRadius: 1, boxShadow: 1 }}>
+ <Typography variant="caption" color="text.secondary">
+ Rotate: drag β€’ Zoom: scroll β€’ Pan: right-drag or two-finger
+ </Typography>
+ </Box>
+
+ <Box sx={{ position: 'absolute', bottom: 16, right: 16, display: 'flex', gap: 1 }}>
+ <Tooltip title="Fit camera to skeleton (F)">
+ <IconButton
+ onClick={handleFitToSkeleton}
+ size="small"
+ sx={{
+ bgcolor: 'background.paper',
+ boxShadow: 2,
+ '&:hover': { bgcolor: 'action.hover' },
+ }}
+ >
+ <CenterFocusStrongIcon fontSize="small" />
+ </IconButton>
+ </Tooltip>
+
+ <Tooltip title="Reset view">
+ <IconButton
+ onClick={() => {
+ const cam = cameraControlsRef.current?.camera;
+ if (cam) {
+ cam.position.set(1e3, 1e3, 1e3);
+ cam.lookAt(0, 0, 0);
+ cam.updateProjectionMatrix();
+ }
+ cameraControlsRef.current?.reset(true);
+ }}
+ size="small"
+ sx={{
+ bgcolor: 'background.paper',
+ boxShadow: 2,
+ '&:hover': { bgcolor: 'action.hover' },
+ }}
+ >
+ <HomeIcon fontSize="small" />
+ </IconButton>
+ </Tooltip>
+ </Box>
+ </Box>
+ );
+}
diff --git a/freemocap-ui/src/components/old-viewport3d/ThreeJsScene.tsx b/freemocap-ui/src/components/old-viewport3d/ThreeJsScene.tsx
new file mode 100644
index 00000000..e1d51668
--- /dev/null
+++ b/freemocap-ui/src/components/old-viewport3d/ThreeJsScene.tsx
@@ -0,0 +1,22 @@
+import {RefObject} from "react";
+import {CameraControls} from "@react-three/drei";
+import type CameraControlsImpl from "camera-controls";
+import {SceneEnvironment} from "@/components/viewport3d/scene/SceneEnvironment";
+import {SkeletonRenderer} from "@/components/viewport3d/SkeletonRenderer";
+import {RigidBodyRenderer} from "@/components/viewport3d/renderers/RigidBodyRenderer";
+
+interface ThreeJsSceneProps {
+ cameraControlsRef: RefObject<CameraControlsImpl>;
+}
+
+/** Top-level scene graph: camera controls, environment, and skeleton visualization. */
+export function ThreeJsScene({ cameraControlsRef }: ThreeJsSceneProps) {
+ return (
+ <>
+ <CameraControls ref={cameraControlsRef} makeDefault />
+ <SceneEnvironment />
+ <SkeletonRenderer />
+ {/*<RigidBodyRenderer />*/}
+ </>
+ );
+}
diff --git a/freemocap-ui/src/components/viewport3d/face-contours.ts b/freemocap-ui/src/components/old-viewport3d/face-contours.ts
similarity index 100%
rename from freemocap-ui/src/components/viewport3d/face-contours.ts
rename to freemocap-ui/src/components/old-viewport3d/face-contours.ts
diff --git a/freemocap-ui/src/components/viewport3d/fit-camera.ts b/freemocap-ui/src/components/old-viewport3d/fit-camera.ts
similarity index 100%
rename from freemocap-ui/src/components/viewport3d/fit-camera.ts
rename to freemocap-ui/src/components/old-viewport3d/fit-camera.ts
diff --git a/freemocap-ui/src/components/viewport3d/skeleton-colors.ts b/freemocap-ui/src/components/old-viewport3d/skeleton-colors.ts
similarity index 100%
rename from freemocap-ui/src/components/viewport3d/skeleton-colors.ts
rename to freemocap-ui/src/components/old-viewport3d/skeleton-colors.ts
diff --git a/freemocap-ui/src/components/viewport3d/skeleton-config.ts b/freemocap-ui/src/components/old-viewport3d/skeleton-config.ts
similarity index 98%
rename from freemocap-ui/src/components/viewport3d/skeleton-config.ts
rename to freemocap-ui/src/components/old-viewport3d/skeleton-config.ts
index 256e688f..4ba8d981 100644
--- a/freemocap-ui/src/components/viewport3d/skeleton-config.ts
+++ b/freemocap-ui/src/components/old-viewport3d/skeleton-config.ts
@@ -1,6 +1,6 @@
import {Color} from "three";
-import {PointStyle} from "@/components/viewport3d/viewport3d-types";
-import {buildFaceContourColors, buildFaceContourSegments} from "@/components/viewport3d/face-contours";
+import {PointStyle} from "@/components/viewport3d/helpers/viewport3d-types";
+import {buildFaceContourColors, buildFaceContourSegments} from "@/components/viewport3d/helpers/face-contours";
import {SKELETON_COLORS} from "@/components/viewport3d/skeleton-colors";
// Maximum tracked points the instanced mesh can hold
diff --git a/freemocap-ui/src/components/viewport3d/viewport3d-types.ts b/freemocap-ui/src/components/old-viewport3d/viewport3d-types.ts
similarity index 100%
rename from freemocap-ui/src/components/viewport3d/viewport3d-types.ts
rename to freemocap-ui/src/components/old-viewport3d/viewport3d-types.ts
diff --git a/freemocap-ui/src/components/viewport3d/virtual-points.ts b/freemocap-ui/src/components/old-viewport3d/virtual-points.ts
similarity index 94%
rename from freemocap-ui/src/components/viewport3d/virtual-points.ts
rename to freemocap-ui/src/components/old-viewport3d/virtual-points.ts
index 1daf859f..85c8348b 100644
--- a/freemocap-ui/src/components/viewport3d/virtual-points.ts
+++ b/freemocap-ui/src/components/old-viewport3d/virtual-points.ts
@@ -1,4 +1,4 @@
-import {Point3d} from "@/components/viewport3d/viewport3d-types";
+import {Point3d} from "@/components/viewport3d/helpers/viewport3d-types";
/**
* Resolve a point name to a 3D position. Handles both raw landmark names
diff --git a/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsx b/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsx
index fc10ae91..c7a40e44 100644
--- a/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsx
+++ b/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsx
@@ -1,100 +1,54 @@
-import {useCallback, useEffect, useRef} from "react";
-import {Canvas} from "@react-three/fiber";
-import {Box, IconButton, Tooltip, Typography} from "@mui/material";
-import CenterFocusStrongIcon from "@mui/icons-material/CenterFocusStrong";
-import HomeIcon from "@mui/icons-material/Home";
+import { useCallback, useEffect, useRef } from "react";
+import { Canvas } from "@react-three/fiber";
+import { Box } from "@mui/material";
import type CameraControlsImpl from "camera-controls";
-import {ThreeJsScene} from "@/components/viewport3d/ThreeJsScene";
-import {fitCameraToSkeleton} from "@/components/viewport3d/fit-camera";
-import {useServer} from "@/services";
+import { ThreeJsScene } from "./ThreeJsScene";
+import { ViewportOverlay } from "./scene/ViewportOverlay";
+import { ViewportStateProvider } from "./scene/ViewportStateContext";
+import { fitCameraToPoints } from "./scene/SceneCamera";
+import { useServer } from "@/services";
export function ThreeJsCanvas() {
- const cameraControlsRef = useRef<CameraControlsImpl>(null!);
+ const controlsRef = useRef<CameraControlsImpl>(null!);
const containerRef = useRef<HTMLDivElement>(null);
- const { getLatestTrackedPoints } = useServer();
+ const { getLatestKeypointsRaw } = useServer();
- const handleFitToSkeleton = useCallback(() => {
- fitCameraToSkeleton(cameraControlsRef.current, getLatestTrackedPoints());
- }, [getLatestTrackedPoints]);
+ const handleFit = useCallback(() => {
+ fitCameraToPoints(controlsRef.current, getLatestKeypointsRaw());
+ }, [getLatestKeypointsRaw]);
- // Press "F" while the viewport container is focused to fit camera to skeleton
- useEffect(() => {
- const container = containerRef.current;
- if (!container) return;
+ const handleReset = useCallback(() => {
+ controlsRef.current?.reset(true);
+ }, []);
- const handleKeyDown = (e: KeyboardEvent) => {
- if (e.key === "f" || e.key === "F") {
- e.preventDefault();
- handleFitToSkeleton();
- }
+ // "F" key shortcut
+ useEffect(() => {
+ const el = containerRef.current;
+ if (!el) return;
+ const onKey = (e: KeyboardEvent) => {
+ if (e.key === "f" || e.key === "F") { e.preventDefault(); handleFit(); }
};
-
- container.addEventListener("keydown", handleKeyDown);
- return () => container.removeEventListener("keydown", handleKeyDown);
- }, [handleFitToSkeleton]);
+ el.addEventListener("keydown", onKey);
+ return () => el.removeEventListener("keydown", onKey);
+ }, [handleFit]);
return (
- <Box
- ref={containerRef}
- tabIndex={0}
- sx={{
- width: '100%',
- height: '100%',
- position: 'relative',
- outline: 'none',
- }}
- >
- <Canvas
- shadows
- camera={{ position: [1e3, 1e3, 1e3], fov: 75, near: 0.1, far: 10000 }}
- gl={{ antialias: true, logarithmicDepthBuffer: true }}
+ <ViewportStateProvider>
+ <Box
+ ref={containerRef}
+ tabIndex={0}
+ sx={{ width: "100%", height: "100%", position: "relative", outline: "none" }}
>
- <ThreeJsScene cameraControlsRef={cameraControlsRef} />
- </Canvas>
-
- <Box sx={{ position: 'absolute', left: 16, bottom: 16, bgcolor: 'background.paper', p: 1, borderRadius: 1, boxShadow: 1 }}>
- <Typography variant="caption" color="text.secondary">
- Rotate: drag β€’ Zoom: scroll β€’ Pan: right-drag or two-finger
- </Typography>
- </Box>
-
- <Box sx={{ position: 'absolute', bottom: 16, right: 16, display: 'flex', gap: 1 }}>
- <Tooltip title="Fit camera to skeleton (F)">
- <IconButton
- onClick={handleFitToSkeleton}
- size="small"
- sx={{
- bgcolor: 'background.paper',
- boxShadow: 2,
- '&:hover': { bgcolor: 'action.hover' },
- }}
- >
- <CenterFocusStrongIcon fontSize="small" />
- </IconButton>
- </Tooltip>
-
- <Tooltip title="Reset view">
- <IconButton
- onClick={() => {
- const cam = cameraControlsRef.current?.camera;
- if (cam) {
- cam.position.set(1e3, 1e3, 1e3);
- cam.lookAt(0, 0, 0);
- cam.updateProjectionMatrix();
- }
- cameraControlsRef.current?.reset(true);
- }}
- size="small"
- sx={{
- bgcolor: 'background.paper',
- boxShadow: 2,
- '&:hover': { bgcolor: 'action.hover' },
- }}
- >
- <HomeIcon fontSize="small" />
- </IconButton>
- </Tooltip>
+ <Canvas
+ shadows
+ camera={{ position: [1e3, 1e3, 1e3], fov: 75, near: 0.1, far: 1e5 }}
+ gl={{ antialias: true, logarithmicDepthBuffer: true }}
+ >
+ <ThreeJsScene cameraControlsRef={controlsRef} />
+ </Canvas>
+
+ <ViewportOverlay onFitCamera={handleFit} onResetCamera={handleReset} />
</Box>
- </Box>
+ </ViewportStateProvider>
);
}
diff --git a/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsx b/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsx
index e207bca7..576b1de8 100644
--- a/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsx
+++ b/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsx
@@ -1,22 +1,23 @@
import {RefObject} from "react";
-import {CameraControls} from "@react-three/drei";
import type CameraControlsImpl from "camera-controls";
-import {SceneEnvironment} from "@/components/viewport3d/SceneEnvironment";
-import {SkeletonRenderer} from "@/components/viewport3d/SkeletonRenderer";
-import {RigidBodyRenderer} from "@/components/viewport3d/RigidBodyRenderer";
+import {SceneCamera} from "./scene/SceneCamera";
+import {SceneEnvironment} from "./scene/SceneEnvironment";
+import {KeypointsRenderer} from "./renderers/KeypointsRenderer";
interface ThreeJsSceneProps {
cameraControlsRef: RefObject<CameraControlsImpl>;
}
-/** Top-level scene graph: camera controls, environment, and skeleton visualization. */
export function ThreeJsScene({ cameraControlsRef }: ThreeJsSceneProps) {
+ // const { visibility } = useViewportState();
+
return (
<>
- <CameraControls ref={cameraControlsRef} makeDefault />
+ <SceneCamera controlsRef={cameraControlsRef} />
<SceneEnvironment />
- <SkeletonRenderer />
- {/*<RigidBodyRenderer />*/}
+ <KeypointsRenderer />
+ {/*{visibility.rigidBodies && <RigidBodyRenderer />}*/}
+ {/*{visibility.face && <FaceRenderer />}*/}
</>
);
}
diff --git a/freemocap-ui/src/components/viewport3d/helpers/colors.ts b/freemocap-ui/src/components/viewport3d/helpers/colors.ts
new file mode 100644
index 00000000..12b2c05e
--- /dev/null
+++ b/freemocap-ui/src/components/viewport3d/helpers/colors.ts
@@ -0,0 +1,15 @@
+import { Color } from "three";
+
+export const COLORS = {
+ // Keypoints - raw are dimmer/smaller, filtered are brighter/larger
+ raw: new Color("#6688AA"),
+ filtered: new Color("#44FF88"),
+ // Rigid bodies
+ rigidBody: new Color("#CC8833"),
+ rigidBodyX: new Color("#FF4444"),
+ rigidBodyY: new Color("#44CC44"),
+ // Face
+ face: new Color("#FFD700"),
+ // Utility
+ hidden: new Color("#000000"),
+} as const;
diff --git a/freemocap-ui/src/components/viewport3d/helpers/face-contours.ts b/freemocap-ui/src/components/viewport3d/helpers/face-contours.ts
new file mode 100644
index 00000000..f11481cc
--- /dev/null
+++ b/freemocap-ui/src/components/viewport3d/helpers/face-contours.ts
@@ -0,0 +1,78 @@
+/**
+ * MediaPipe face mesh contour connections.
+ * Point names follow `face.{group}_{mediapipe_index}`.
+ */
+
+interface FaceContourGroup {
+ prefix: string;
+ connections: [number, number][];
+}
+
+const FACE_CONTOUR_GROUPS: FaceContourGroup[] = [
+ {
+ prefix: "face.lips",
+ connections: [
+ [61,146],[146,91],[91,181],[181,84],[84,17],[17,314],[314,405],[405,321],[321,375],[375,291],
+ [61,185],[185,40],[40,39],[39,37],[37,0],[0,267],[267,269],[269,270],[270,409],[409,291],
+ [78,95],[95,88],[88,178],[178,87],[87,14],[14,317],[317,402],[402,318],[318,324],[324,308],
+ [78,191],[191,80],[80,81],[81,82],[82,13],[13,312],[312,311],[311,310],[310,415],[415,308],
+ ],
+ },
+ {
+ prefix: "face.left_eye",
+ connections: [
+ [263,249],[249,390],[390,373],[373,374],[374,380],[380,381],[381,382],[382,362],
+ [263,466],[466,388],[388,387],[387,386],[386,385],[385,384],[384,398],[398,362],
+ ],
+ },
+ {
+ prefix: "face.right_eye",
+ connections: [
+ [33,7],[7,163],[163,144],[144,145],[145,153],[153,154],[154,155],[155,133],
+ [33,246],[246,161],[161,160],[160,159],[159,158],[158,157],[157,173],[173,133],
+ ],
+ },
+ {
+ prefix: "face.left_eyebrow",
+ connections: [[276,283],[283,282],[282,295],[295,285],[300,293],[293,334],[334,296],[296,336]],
+ },
+ {
+ prefix: "face.right_eyebrow",
+ connections: [[46,53],[53,52],[52,65],[65,55],[70,63],[63,105],[105,66],[66,107]],
+ },
+ {
+ prefix: "face.face_oval",
+ connections: [
+ [10,338],[338,297],[297,332],[332,284],[284,251],[251,389],[389,356],[356,454],
+ [454,323],[323,361],[361,288],[288,397],[397,365],[365,379],[379,378],[378,400],
+ [400,377],[377,152],[152,148],[148,176],[176,149],[149,150],[150,136],[136,172],
+ [172,58],[58,132],[132,93],[93,234],[234,127],[127,162],[162,21],[21,54],[54,103],
+ [103,67],[67,109],[109,10],
+ ],
+ },
+ {
+ prefix: "face.left_iris",
+ connections: [[474,475],[475,476],[476,477],[477,474]],
+ },
+ {
+ prefix: "face.right_iris",
+ connections: [[469,470],[470,471],[471,472],[472,469]],
+ },
+];
+
+export interface FaceSegment {
+ from: string;
+ to: string;
+}
+
+/** All face contour segments as pairs of point names. */
+export const FACE_SEGMENTS: FaceSegment[] = [];
+
+for (const group of FACE_CONTOUR_GROUPS) {
+ for (const [a, b] of group.connections) {
+ FACE_SEGMENTS.push({
+ from: `${group.prefix}_${a}`,
+ to: `${group.prefix}_${b}`,
+ });
+ }
+}
diff --git a/freemocap-ui/src/components/viewport3d/helpers/viewport3d-types.ts b/freemocap-ui/src/components/viewport3d/helpers/viewport3d-types.ts
new file mode 100644
index 00000000..78bbe957
--- /dev/null
+++ b/freemocap-ui/src/components/viewport3d/helpers/viewport3d-types.ts
@@ -0,0 +1,41 @@
+import { Color } from "three";
+
+/** 3D point */
+export interface Point3d {
+ x: number;
+ y: number;
+ z: number;
+}
+
+/** Rigid body pose from Python backend. */
+export interface RigidBodyPose {
+ bone_key: string;
+ position: [number, number, number];
+ orientation: [number, number, number, number]; // (w, x, y, z)
+ scale: [number, number, number]; // (sx, sy, sz)
+}
+
+/** Viewport layer visibility toggles. */
+export interface ViewportVisibility {
+ environment: boolean;
+ keypointsRaw: boolean;
+ keypointsFiltered: boolean;
+ rigidBodies: boolean;
+ face: boolean;
+}
+
+export const DEFAULT_VISIBILITY: ViewportVisibility = {
+ environment: true,
+ keypointsRaw: true,
+ keypointsFiltered: true,
+ rigidBodies: true,
+ face: true,
+};
+
+/** Live stats from each renderer. */
+export interface ViewportStats {
+ keypointsRaw: number;
+ keypointsFiltered: number;
+ rigidBodies: number;
+ facePoints: number;
+}
diff --git a/freemocap-ui/src/components/viewport3d/index.ts b/freemocap-ui/src/components/viewport3d/index.ts
new file mode 100644
index 00000000..981d348e
--- /dev/null
+++ b/freemocap-ui/src/components/viewport3d/index.ts
@@ -0,0 +1,2 @@
+export { ThreeJsCanvas } from "./ThreeJsCanvas";
+export type { Point3d, RigidBodyPose, ViewportVisibility } from "./helpers/viewport3d-types";
diff --git a/freemocap-ui/src/components/viewport3d/renderers/FaceRenderer.tsx b/freemocap-ui/src/components/viewport3d/renderers/FaceRenderer.tsx
new file mode 100644
index 00000000..ce6fafb0
--- /dev/null
+++ b/freemocap-ui/src/components/viewport3d/renderers/FaceRenderer.tsx
@@ -0,0 +1,136 @@
+import { useEffect, useMemo, useRef } from "react";
+import {
+ BufferAttribute,
+ BufferGeometry,
+ LineBasicMaterial,
+ LineSegments,
+ InstancedMesh,
+ MeshBasicMaterial,
+ Object3D,
+ SphereGeometry,
+ Vector3,
+} from "three";
+import { useFrame } from "@react-three/fiber";
+import { useServer } from "@/services";
+import { Point3d } from "../helpers/viewport3d-types";
+import { useViewportState } from "../scene/ViewportStateContext";
+import { COLORS } from "../helpers/colors";
+import { FACE_SEGMENTS } from "../helpers/face-contours";
+
+const MAX_FACE_POINTS = 512;
+const DOT_RADIUS = 0.015;
+const DUMMY = new Object3D();
+const FAR_AWAY = new Vector3(1e5, 1e5, 1e5);
+
+/**
+ * Renders face mesh as small dots + contour line segments.
+ * Subscribes to raw keypoints and filters for face.* names.
+ */
+export function FaceRenderer() {
+ const { subscribeToKeypointsRaw } = useServer();
+ const { statsRef } = useViewportState();
+
+ const dotsRef = useRef<InstancedMesh>(null);
+ const linesRef = useRef<LineSegments>(null);
+
+ const pointsRef = useRef<Map<string, Point3d>>(new Map());
+ const nameToIdx = useRef<Map<string, number>>(new Map());
+ const nextIdx = useRef(0);
+ const dirtyRef = useRef(false);
+
+ const dotGeo = useMemo(() => new SphereGeometry(1, 4, 3), []);
+ const dotMat = useMemo(() => new MeshBasicMaterial({ color: COLORS.face }), []);
+
+ const lineMat = useMemo(() => new LineBasicMaterial({
+ color: COLORS.face,
+ transparent: true,
+ opacity: 0.7,
+ }), []);
+
+ const lineGeo = useMemo(() => {
+ const g = new BufferGeometry();
+ const n = FACE_SEGMENTS.length * 2;
+ const pos = new Float32Array(n * 3).fill(1e5);
+ g.setAttribute("position", new BufferAttribute(pos, 3));
+ return g;
+ }, []);
+
+ useEffect(() => {
+ return subscribeToKeypointsRaw((allPts: Map<string, Point3d>) => {
+ const face = new Map<string, Point3d>();
+ for (const [name, pt] of allPts) {
+ if (name.startsWith("face.")) face.set(name, pt);
+ }
+ pointsRef.current = face;
+ dirtyRef.current = true;
+
+ for (const name of face.keys()) {
+ if (!nameToIdx.current.has(name) && nextIdx.current < MAX_FACE_POINTS) {
+ nameToIdx.current.set(name, nextIdx.current++);
+ }
+ }
+ });
+ }, [subscribeToKeypointsRaw]);
+
+ useEffect(() => {
+ const mesh = dotsRef.current;
+ if (!mesh) return;
+ for (let i = 0; i < MAX_FACE_POINTS; i++) {
+ DUMMY.position.copy(FAR_AWAY);
+ DUMMY.scale.set(0, 0, 0);
+ DUMMY.updateMatrix();
+ mesh.setMatrixAt(i, DUMMY.matrix);
+ }
+ mesh.instanceMatrix.needsUpdate = true;
+ mesh.count = MAX_FACE_POINTS;
+ }, []);
+
+ useFrame(() => {
+ const dots = dotsRef.current;
+ if (!dots || !dirtyRef.current) return;
+
+ const pts = pointsRef.current;
+ let count = 0;
+
+ for (const [name, idx] of nameToIdx.current) {
+ const pt = pts.get(name);
+ if (pt) {
+ DUMMY.position.set(pt.x, pt.y, pt.z);
+ DUMMY.scale.setScalar(DOT_RADIUS);
+ count++;
+ } else {
+ DUMMY.position.copy(FAR_AWAY);
+ DUMMY.scale.set(0, 0, 0);
+ }
+ DUMMY.updateMatrix();
+ dots.setMatrixAt(idx, DUMMY.matrix);
+ }
+ dots.instanceMatrix.needsUpdate = true;
+
+ // Update line segments
+ const positions = lineGeo.attributes.position.array as Float32Array;
+ for (let i = 0; i < FACE_SEGMENTS.length; i++) {
+ const seg = FACE_SEGMENTS[i];
+ const a = pts.get(seg.from);
+ const b = pts.get(seg.to);
+ const base = i * 6;
+ if (a && b) {
+ positions[base] = a.x; positions[base+1] = a.y; positions[base+2] = a.z;
+ positions[base+3] = b.x; positions[base+4] = b.y; positions[base+5] = b.z;
+ } else {
+ for (let j = 0; j < 6; j++) positions[base + j] = 1e5;
+ }
+ }
+ lineGeo.attributes.position.needsUpdate = true;
+
+ dirtyRef.current = false;
+ statsRef.current.facePoints = count;
+ });
+
+ return (
+ <>
+ <instancedMesh ref={dotsRef} args={[dotGeo, dotMat, MAX_FACE_POINTS]} frustumCulled={false} />
+ <lineSegments ref={linesRef} geometry={lineGeo} material={lineMat} frustumCulled={false} />
+ </>
+ );
+}
diff --git a/freemocap-ui/src/components/viewport3d/renderers/KeypointsRenderer.tsx b/freemocap-ui/src/components/viewport3d/renderers/KeypointsRenderer.tsx
new file mode 100644
index 00000000..34a6b48c
--- /dev/null
+++ b/freemocap-ui/src/components/viewport3d/renderers/KeypointsRenderer.tsx
@@ -0,0 +1,131 @@
+import { useEffect, useMemo, useRef } from "react";
+import {
+ Color,
+ InstancedMesh,
+ MeshStandardMaterial,
+ Object3D,
+ SphereGeometry,
+ Vector3,
+} from "three";
+import { useFrame } from "@react-three/fiber";
+import { useServer } from "@/services";
+import { Point3d } from "../helpers/viewport3d-types";
+import { useViewportState } from "../scene/ViewportStateContext";
+import { COLORS } from "../helpers/colors";
+
+const MAX_POINTS = 1024;
+const DUMMY = new Object3D();
+const FAR_AWAY = new Vector3(1e5, 1e5, 1e5);
+
+const RAW_RADIUS = 0.08;
+const FILTERED_RADIUS = 0.12;
+
+interface KeypointLayerProps {
+ subscribeKey: "subscribeToKeypointsRaw" | "subscribeToKeypointsFiltered";
+ color: Color;
+ radius: number;
+ statsKey: "keypointsRaw" | "keypointsFiltered";
+}
+
+function KeypointLayer({ subscribeKey, color, radius, statsKey }: KeypointLayerProps) {
+ const server = useServer();
+ const { statsRef } = useViewportState();
+ const meshRef = useRef<InstancedMesh>(null);
+ const pointsRef = useRef<Record<string, Point3d>>({}); // plain object, matches wire format
+ const dirtyRef = useRef(false);
+ const nameToIdx = useRef<Map<string, number>>(new Map());
+ const nextIdx = useRef(0);
+
+ const geo = useMemo(() => new SphereGeometry(100, 8, 6), []);
+ const mat = useMemo(() => new MeshStandardMaterial({
+ color: "#ffffff",
+ roughness: 0.4,
+ metalness: 0.3,
+ }), []);
+
+ useEffect(() => {
+ const subscribeFn = server[subscribeKey];
+ return subscribeFn((pts: Record<string, Point3d>) => {
+ pointsRef.current = pts;
+ dirtyRef.current = true;
+ for (const name of Object.keys(pts)) { // was: pts.keys()
+ if (!nameToIdx.current.has(name) && nextIdx.current < MAX_POINTS) {
+ nameToIdx.current.set(name, nextIdx.current++);
+ }
+ }
+ });
+ }, [server, subscribeKey]);
+
+ useEffect(() => {
+ const mesh = meshRef.current;
+ if (!mesh) return;
+ for (let i = 0; i < MAX_POINTS; i++) {
+ DUMMY.position.copy(FAR_AWAY);
+ DUMMY.scale.set(0, 0, 0);
+ DUMMY.updateMatrix();
+ mesh.setMatrixAt(i, DUMMY.matrix);
+ mesh.setColorAt(i, COLORS.hidden);
+ }
+ mesh.instanceMatrix.needsUpdate = true;
+ if (mesh.instanceColor) mesh.instanceColor.needsUpdate = true;
+ mesh.count = MAX_POINTS;
+ }, []);
+
+ useFrame(() => {
+ const mesh = meshRef.current;
+ if (!mesh || !dirtyRef.current) return;
+ const pts = pointsRef.current;
+ let count = 0;
+
+ for (const [name, idx] of nameToIdx.current) {
+ const pt = pts[name]; // was: pts.get(name)
+ if (pt) {
+ DUMMY.position.set(pt.x, pt.y, pt.z);
+ DUMMY.scale.setScalar(radius);
+ mesh.setColorAt(idx, color);
+ count++;
+ } else {
+ // Not in this packet β€” hide it
+ DUMMY.position.copy(FAR_AWAY);
+ DUMMY.scale.set(0, 0, 0);
+ mesh.setColorAt(idx, COLORS.hidden);
+ }
+ DUMMY.updateMatrix();
+ mesh.setMatrixAt(idx, DUMMY.matrix);
+ }
+
+ mesh.instanceMatrix.needsUpdate = true;
+ if (mesh.instanceColor) mesh.instanceColor.needsUpdate = true;
+ dirtyRef.current = false;
+ statsRef.current[statsKey] = count;
+ });
+
+ return (
+ <instancedMesh ref={meshRef} args={[geo, mat, MAX_POINTS]} frustumCulled={false} />
+ );
+}
+
+export function KeypointsRenderer() {
+ const { visibility } = useViewportState();
+
+ return (
+ <>
+ {visibility.keypointsRaw && (
+ <KeypointLayer
+ subscribeKey="subscribeToKeypointsRaw"
+ color={COLORS.raw}
+ radius={RAW_RADIUS}
+ statsKey="keypointsRaw"
+ />
+ )}
+ {visibility.keypointsFiltered && (
+ <KeypointLayer
+ subscribeKey="subscribeToKeypointsFiltered"
+ color={COLORS.filtered}
+ radius={FILTERED_RADIUS}
+ statsKey="keypointsFiltered"
+ />
+ )}
+ </>
+ );
+}
diff --git a/freemocap-ui/src/components/viewport3d/renderers/RigidBodyRenderer.tsx b/freemocap-ui/src/components/viewport3d/renderers/RigidBodyRenderer.tsx
new file mode 100644
index 00000000..b1c7a0e4
--- /dev/null
+++ b/freemocap-ui/src/components/viewport3d/renderers/RigidBodyRenderer.tsx
@@ -0,0 +1,164 @@
+import { useEffect, useMemo, useRef } from "react";
+import {
+ CylinderGeometry,
+ Group,
+ InstancedMesh,
+ MeshStandardMaterial,
+ Object3D,
+ Vector3,
+} from "three";
+import { useFrame } from "@react-three/fiber";
+import { useServer } from "@/services";
+import { RigidBodyPose } from "../helpers/viewport3d-types";
+import { useViewportState } from "../scene/ViewportStateContext";
+import { COLORS } from "../helpers/colors";
+
+const MAX_BODIES = 128;
+const DUMMY = new Object3D();
+const FAR_AWAY = new Vector3(1e5, 1e5, 1e5);
+
+/** Shaft radius for the main Z-axis cylinder. */
+const SHAFT_RADIUS = 0.012;
+/** Crossbar radius and length for X/Y axis indicators. */
+const CROSSBAR_RADIUS = 0.006;
+const CROSSBAR_LENGTH = 0.04;
+
+/**
+ * Renders rigid body segments as instanced cylinders along local Z,
+ * with small crossbars at the origin showing X (red) and Y (green) axes.
+ *
+ * Uses 3 instanced meshes:
+ * - main shafts (Z axis, bone color)
+ * - X crossbars (red)
+ * - Y crossbars (green)
+ */
+export function RigidBodyRenderer() {
+ const { subscribeToRigidBodies } = useServer();
+ const { statsRef } = useViewportState();
+
+ const shaftRef = useRef<InstancedMesh>(null);
+ const xBarRef = useRef<InstancedMesh>(null);
+ const yBarRef = useRef<InstancedMesh>(null);
+
+ const posesRef = useRef<Map<string, RigidBodyPose>>(new Map());
+ const nameToIdx = useRef<Map<string, number>>(new Map());
+ const nextIdx = useRef(0);
+ const dirtyRef = useRef(false);
+
+ // Shaft geometry: unit cylinder along +Z, origin at bottom
+ const shaftGeo = useMemo(() => {
+ const g = new CylinderGeometry(1, 1, 1, 6, 1);
+ g.rotateX(Math.PI / 2); // Y β†’ Z
+ g.translate(0, 0, 0.5); // origin at bottom
+ return g;
+ }, []);
+
+ // Crossbar geometry: short cylinder along +X (will be rotated for Y)
+ const crossbarGeo = useMemo(() => {
+ const g = new CylinderGeometry(1, 1, 1, 4, 1);
+ // leave along Y β€” we'll orient per-instance
+ return g;
+ }, []);
+
+ const shaftMat = useMemo(() => new MeshStandardMaterial({
+ color: "#ffffff", roughness: 0.5, metalness: 0.2, transparent: true, opacity: 0.7,
+ }), []);
+ const xMat = useMemo(() => new MeshStandardMaterial({ color: COLORS.rigidBodyX }), []);
+ const yMat = useMemo(() => new MeshStandardMaterial({ color: COLORS.rigidBodyY }), []);
+
+ useEffect(() => {
+ return subscribeToRigidBodies((newPoses: Map<string, RigidBodyPose>) => {
+ posesRef.current = newPoses;
+ dirtyRef.current = true;
+ for (const key of newPoses.keys()) {
+ if (!nameToIdx.current.has(key) && nextIdx.current < MAX_BODIES) {
+ nameToIdx.current.set(key, nextIdx.current++);
+ }
+ }
+ });
+ }, [subscribeToRigidBodies]);
+
+ // Init all hidden
+ useEffect(() => {
+ for (const mesh of [shaftRef.current, xBarRef.current, yBarRef.current]) {
+ if (!mesh) continue;
+ for (let i = 0; i < MAX_BODIES; i++) {
+ DUMMY.position.copy(FAR_AWAY);
+ DUMMY.scale.set(0, 0, 0);
+ DUMMY.updateMatrix();
+ mesh.setMatrixAt(i, DUMMY.matrix);
+ if (mesh === shaftRef.current) mesh.setColorAt(i, COLORS.hidden);
+ }
+ mesh.instanceMatrix.needsUpdate = true;
+ if (mesh.instanceColor) mesh.instanceColor.needsUpdate = true;
+ mesh.count = MAX_BODIES;
+ }
+ }, []);
+
+ useFrame(() => {
+ const shaft = shaftRef.current;
+ const xBar = xBarRef.current;
+ const yBar = yBarRef.current;
+ if (!shaft || !xBar || !yBar || !dirtyRef.current) return;
+
+ const poses = posesRef.current;
+ let count = 0;
+
+ for (const [key, idx] of nameToIdx.current) {
+ const pose = poses.get(key);
+
+ if (pose) {
+ const [px, py, pz] = pose.position;
+ const [qw, qx, qy, qz] = pose.orientation;
+ const [sx, sy, sz] = pose.scale;
+
+ // Main shaft along local Z
+ DUMMY.position.set(px, py, pz);
+ DUMMY.quaternion.set(qx, qy, qz, qw);
+ DUMMY.scale.set(SHAFT_RADIUS, SHAFT_RADIUS, sz);
+ DUMMY.updateMatrix();
+ shaft.setMatrixAt(idx, DUMMY.matrix);
+ shaft.setColorAt(idx, COLORS.rigidBody);
+
+ // X crossbar: rotate 90Β° around Z from the bone frame, scale along local Y
+ DUMMY.scale.set(CROSSBAR_RADIUS, CROSSBAR_LENGTH, CROSSBAR_RADIUS);
+ DUMMY.updateMatrix();
+ // We need to apply an additional 90Β° rotation around Z for the X bar
+ // Simpler: just set scale to put length along X
+ DUMMY.scale.set(CROSSBAR_LENGTH, CROSSBAR_RADIUS, CROSSBAR_RADIUS);
+ DUMMY.updateMatrix();
+ xBar.setMatrixAt(idx, DUMMY.matrix);
+
+ // Y crossbar
+ DUMMY.scale.set(CROSSBAR_RADIUS, CROSSBAR_LENGTH, CROSSBAR_RADIUS);
+ DUMMY.updateMatrix();
+ yBar.setMatrixAt(idx, DUMMY.matrix);
+
+ count++;
+ } else {
+ DUMMY.position.copy(FAR_AWAY);
+ DUMMY.scale.set(0, 0, 0);
+ DUMMY.updateMatrix();
+ shaft.setMatrixAt(idx, DUMMY.matrix);
+ shaft.setColorAt(idx, COLORS.hidden);
+ xBar.setMatrixAt(idx, DUMMY.matrix);
+ yBar.setMatrixAt(idx, DUMMY.matrix);
+ }
+ }
+
+ for (const mesh of [shaft, xBar, yBar]) {
+ mesh.instanceMatrix.needsUpdate = true;
+ if (mesh.instanceColor) mesh.instanceColor.needsUpdate = true;
+ }
+ dirtyRef.current = false;
+ statsRef.current.rigidBodies = count;
+ });
+
+ return (
+ <>
+ <instancedMesh ref={shaftRef} args={[shaftGeo, shaftMat, MAX_BODIES]} frustumCulled={false} />
+ <instancedMesh ref={xBarRef} args={[crossbarGeo, xMat, MAX_BODIES]} frustumCulled={false} />
+ <instancedMesh ref={yBarRef} args={[crossbarGeo, yMat, MAX_BODIES]} frustumCulled={false} />
+ </>
+ );
+}
diff --git a/freemocap-ui/src/components/viewport3d/scene/SceneCamera.tsx b/freemocap-ui/src/components/viewport3d/scene/SceneCamera.tsx
new file mode 100644
index 00000000..b1847b9d
--- /dev/null
+++ b/freemocap-ui/src/components/viewport3d/scene/SceneCamera.tsx
@@ -0,0 +1,31 @@
+import { useCallback, useEffect } from "react";
+import { CameraControls } from "@react-three/drei";
+import type CameraControlsImpl from "camera-controls";
+import { Box3, Vector3 } from "three";
+import { useServer } from "@/services";
+import { Point3d } from "../helpers/viewport3d-types";
+import { RefObject } from "react";
+
+const PADDING = 0.5;
+
+interface SceneCameraProps {
+ controlsRef: RefObject<CameraControlsImpl>;
+}
+
+export function SceneCamera({ controlsRef }: SceneCameraProps) {
+ return <CameraControls ref={controlsRef} makeDefault />;
+}
+
+/** Fit camera to bounding box of all keypoints. */
+export function fitCameraToPoints(
+ controls: CameraControlsImpl | null,
+ points: Map<string, Point3d>,
+): void {
+ if (!controls || points.size === 0) return;
+ const box = new Box3();
+ for (const pt of points.values()) {
+ box.expandByPoint(new Vector3(pt.x, pt.y, pt.z));
+ }
+ box.expandByScalar(PADDING);
+ controls.fitToBox(box, true);
+}
diff --git a/freemocap-ui/src/components/viewport3d/scene/SceneEnvironment.tsx b/freemocap-ui/src/components/viewport3d/scene/SceneEnvironment.tsx
new file mode 100644
index 00000000..86f50376
--- /dev/null
+++ b/freemocap-ui/src/components/viewport3d/scene/SceneEnvironment.tsx
@@ -0,0 +1,28 @@
+import { Grid } from "@react-three/drei";
+import { useViewportState } from "./ViewportStateContext";
+
+export function SceneEnvironment() {
+ const { visibility } = useViewportState();
+ if (!visibility.environment) return null;
+
+ return (
+ <>
+ <ambientLight intensity={0.6} />
+ <directionalLight position={[3e3, 3e3, 3e3]} intensity={0.5} />
+ <directionalLight position={[-3e3, -2e3, -3e3]} intensity={0.3} />
+ <Grid
+ renderOrder={-1}
+ position={[0, -0.01, 0]}
+ infiniteGrid
+ cellSize={1000}
+ cellThickness={0.5}
+ sectionSize={50}
+ sectionThickness={1}
+ // @ts-ignore
+ sectionColor={[0.1, 0, 0.1]}
+ fadeDistance={10000}
+ />
+ <axesHelper scale={100} />
+ </>
+ );
+}
diff --git a/freemocap-ui/src/components/viewport3d/scene/ViewportOverlay.tsx b/freemocap-ui/src/components/viewport3d/scene/ViewportOverlay.tsx
new file mode 100644
index 00000000..b27040f8
--- /dev/null
+++ b/freemocap-ui/src/components/viewport3d/scene/ViewportOverlay.tsx
@@ -0,0 +1,112 @@
+import { useCallback, useEffect, useState } from "react";
+import {
+ Box,
+ Checkbox,
+ Collapse,
+ FormControlLabel,
+ IconButton,
+ Paper,
+ Tooltip,
+ Typography,
+} from "@mui/material";
+import ExpandMoreIcon from "@mui/icons-material/ExpandMore";
+import ExpandLessIcon from "@mui/icons-material/ExpandLess";
+import CenterFocusStrongIcon from "@mui/icons-material/CenterFocusStrong";
+import HomeIcon from "@mui/icons-material/Home";
+import { ViewportStats, ViewportVisibility } from "../helpers/viewport3d-types";
+import { useViewportState } from "./ViewportStateContext";
+
+interface ViewportOverlayProps {
+ onFitCamera: () => void;
+ onResetCamera: () => void;
+}
+
+export function ViewportOverlay({ onFitCamera, onResetCamera }: ViewportOverlayProps) {
+ const { visibility, setVisibility, statsRef } = useViewportState();
+ const [stats, setStats] = useState<ViewportStats>({ keypointsRaw: 0, keypointsFiltered: 0, rigidBodies: 0, facePoints: 0 });
+ const [expanded, setExpanded] = useState(false);
+
+ // Poll stats from the mutable ref at ~4 Hz (no re-render coupling to the frame loop)
+ useEffect(() => {
+ const id = setInterval(() => setStats({ ...statsRef.current }), 250);
+ return () => clearInterval(id);
+ }, [statsRef]);
+
+ const toggle = useCallback((key: keyof ViewportVisibility) => {
+ setVisibility(prev => ({ ...prev, [key]: !prev[key] }));
+ }, [setVisibility]);
+
+ return (
+ <>
+ {/* Top-left: visibility + stats */}
+ <Paper
+ sx={{
+ position: "absolute", top: 8, left: 8,
+ p: 1, bgcolor: "rgba(0,0,0,0.75)", color: "#ccc",
+ minWidth: 180, userSelect: "none",
+ }}
+ elevation={3}
+ >
+ <Box sx={{ display: "flex", alignItems: "center", justifyContent: "space-between" }}>
+ <Typography variant="caption" fontWeight="bold">Viewport</Typography>
+ <IconButton size="small" onClick={() => setExpanded(e => !e)} sx={{ color: "#ccc" }}>
+ {expanded ? <ExpandLessIcon fontSize="small" /> : <ExpandMoreIcon fontSize="small" />}
+ </IconButton>
+ </Box>
+
+ <VisToggle label="Environment" checked={visibility.environment} onChange={() => toggle("environment")} />
+ <VisToggle label={`Raw (${stats.keypointsRaw})`} checked={visibility.keypointsRaw} onChange={() => toggle("keypointsRaw")} />
+ <VisToggle label={`Filtered (${stats.keypointsFiltered})`} checked={visibility.keypointsFiltered} onChange={() => toggle("keypointsFiltered")} />
+ <VisToggle label={`Rigid bodies (${stats.rigidBodies})`} checked={visibility.rigidBodies} onChange={() => toggle("rigidBodies")} />
+ <VisToggle label={`Face (${stats.facePoints})`} checked={visibility.face} onChange={() => toggle("face")} />
+
+ <Collapse in={expanded}>
+ <Typography variant="caption" sx={{ mt: 1, display: "block", color: "#888" }}>
+ Total points: {stats.keypointsRaw + stats.keypointsFiltered + stats.facePoints}
+ <br />
+ Total bodies: {stats.rigidBodies}
+ </Typography>
+ </Collapse>
+ </Paper>
+
+ {/* Bottom-right: camera buttons */}
+ <Box sx={{ position: "absolute", bottom: 8, right: 8, display: "flex", gap: 0.5 }}>
+ <Tooltip title="Fit to skeleton (F)">
+ <IconButton onClick={onFitCamera} size="small" sx={btnSx}>
+ <CenterFocusStrongIcon fontSize="small" />
+ </IconButton>
+ </Tooltip>
+ <Tooltip title="Reset view">
+ <IconButton onClick={onResetCamera} size="small" sx={btnSx}>
+ <HomeIcon fontSize="small" />
+ </IconButton>
+ </Tooltip>
+ </Box>
+
+ {/* Bottom-left: hint */}
+ <Typography
+ variant="caption"
+ sx={{ position: "absolute", bottom: 8, left: 8, color: "#666", pointerEvents: "none" }}
+ >
+ Rotate: drag Β· Zoom: scroll Β· Pan: right-drag
+ </Typography>
+ </>
+ );
+}
+
+function VisToggle({ label, checked, onChange }: { label: string; checked: boolean; onChange: () => void }) {
+ return (
+ <FormControlLabel
+ sx={{ m: 0, ml: -0.5, "& .MuiTypography-root": { fontSize: "0.7rem" } }}
+ control={<Checkbox size="small" checked={checked} onChange={onChange} sx={{ p: 0.3, color: "#888", "&.Mui-checked": { color: "#aaa" } }} />}
+ label={label}
+ />
+ );
+}
+
+const btnSx = {
+ bgcolor: "rgba(0,0,0,0.6)",
+ color: "#ccc",
+ boxShadow: 2,
+ "&:hover": { bgcolor: "rgba(60,60,60,0.8)" },
+};
diff --git a/freemocap-ui/src/components/viewport3d/scene/ViewportStateContext.tsx b/freemocap-ui/src/components/viewport3d/scene/ViewportStateContext.tsx
new file mode 100644
index 00000000..ae41ab47
--- /dev/null
+++ b/freemocap-ui/src/components/viewport3d/scene/ViewportStateContext.tsx
@@ -0,0 +1,35 @@
+import React, { createContext, useCallback, useContext, useMemo, useRef, useState } from "react";
+import { DEFAULT_VISIBILITY, ViewportStats, ViewportVisibility } from "../helpers/viewport3d-types";
+
+interface ViewportStateContextValue {
+ visibility: ViewportVisibility;
+ setVisibility: React.Dispatch<React.SetStateAction<ViewportVisibility>>;
+ /** Mutable stats ref β€” renderers write here each frame, overlay reads on a poll timer. */
+ statsRef: React.MutableRefObject<ViewportStats>;
+}
+
+const ViewportStateContext = createContext<ViewportStateContextValue | null>(null);
+
+export function ViewportStateProvider({ children }: { children: React.ReactNode }) {
+ const [visibility, setVisibility] = useState<ViewportVisibility>(DEFAULT_VISIBILITY);
+ const statsRef = useRef<ViewportStats>({
+ keypointsRaw: 0,
+ keypointsFiltered: 0,
+ rigidBodies: 0,
+ facePoints: 0,
+ });
+
+ const value = useMemo(() => ({ visibility, setVisibility, statsRef }), [visibility]);
+
+ return (
+ <ViewportStateContext.Provider value={value}>
+ {children}
+ </ViewportStateContext.Provider>
+ );
+}
+
+export function useViewportState() {
+ const ctx = useContext(ViewportStateContext);
+ if (!ctx) throw new Error("useViewportState must be used within ViewportStateProvider");
+ return ctx;
+}
diff --git a/freemocap-ui/src/services/server/ServerContextProvider.tsx b/freemocap-ui/src/services/server/ServerContextProvider.tsx
index 1693c4d6..66ca14f6 100644
--- a/freemocap-ui/src/services/server/ServerContextProvider.tsx
+++ b/freemocap-ui/src/services/server/ServerContextProvider.tsx
@@ -5,9 +5,8 @@ import {ConnectionState, WebSocketConnection} from "@/services/server/server-hel
import {FrameProcessor} from "@/services/server/server-helpers/frame-processor/frame-processor";
import {CanvasManager} from "@/services/server/server-helpers/canvas-manager";
import {serverUrls} from "@/services";
-import {DetailedFramerate, FramerateStore} from "@/services/server/server-helpers/framerate-store";
-import {LogRecord, LogStore} from "@/services/server/server-helpers/log-store";
-import {Point3d, RigidBodyPose} from "@/components/viewport3d/viewport3d-types";
+import {FramerateStore} from "@/services/server/server-helpers/framerate-store";
+import {LogStore} from "@/services/server/server-helpers/log-store";
import {OverlayManager} from "@/services/server/server-helpers/image-overlay/overlay-renderer-factory";
import {CharucoObservation} from "@/services/server/server-helpers/image-overlay/charuco-types";
import {MediapipeObservation} from "@/services/server/server-helpers/image-overlay/mediapipe-types";
@@ -16,6 +15,7 @@ import {
isFrontendPayload,
isLogRecord
} from "@/services/server/server-helpers/websocket-message-types";
+import {Point3d, RigidBodyPose} from "@/components/viewport3d";
interface ServerContextValue {
isConnected: boolean;
@@ -29,9 +29,10 @@ interface ServerContextValue {
getLogStore: () => LogStore;
connectedCameraIds: string[];
updateServerConnection: (host: string, port: number) => void;
- subscribeToTrackedPoints: (cb: (points: Map<string, Point3d>) => void) => () => void;
+ subscribeToKeypointsRaw: (cb: (points: Record<string, Point3d>) => void) => () => void;
+ subscribeToKeypointsFiltered: (cb: (points: Record<string, Point3d>) => void) => () => void;
subscribeToRigidBodies: (cb: (poses: Map<string, RigidBodyPose>) => void) => () => void;
- getLatestTrackedPoints: () => Map<string, Point3d>;
+ getLatestKeypointsRaw: () => Record<string, Point3d>;
}
const ServerContext = createContext<ServerContextValue | null>(null);
@@ -66,10 +67,13 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({childr
const serverFpsRef = useRef<number | null>(null);
// 3D data refs and subscriber sets
- const trackedPointsRef = useRef<Map<string, Point3d>>(new Map());
+ // Plain Record β€” matches JSON.parse output directly, no remapping needed.
+ const trackedPointsRef = useRef<Record<string, Point3d>>({});
const rigidBodiesRef = useRef<Map<string, RigidBodyPose>>(new Map());
- const trackedPointsSubscribersRef = useRef<Set<(points: Map<string, Point3d>) => void>>(new Set());
+ const trackedPointsSubscribersRef = useRef<Set<(points: Record<string, Point3d>) => void>>(new Set());
const rigidBodiesSubscribersRef = useRef<Set<(poses: Map<string, RigidBodyPose>) => void>>(new Set());
+ const keypointsFilteredRef = useRef<Record<string, Point3d>>({});
+ const keypointsFilteredSubscribersRef = useRef<Set<(points: Record<string, Point3d>) => void>>(new Set());
// Holds the latest binary payload received from the WebSocket.
@@ -131,8 +135,9 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({childr
pendingPayloadRef.current = null;
lastCameraIdsRef.current = [];
framerateStoreRef.current.clear();
- trackedPointsRef.current = new Map();
+ trackedPointsRef.current = {};
rigidBodiesRef.current = new Map();
+ keypointsFilteredRef.current = {};
latestCharucoRef.current.clear();
latestMediapipeRef.current.clear();
overlayManagerRef.current?.clearAll();
@@ -237,14 +242,14 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({childr
if (event.data instanceof ArrayBuffer) {
pendingPayloadRef.current = event.data;
}
- // Handle text/JSON messages (logs, framerate updates, frontend payloads, etc.)
+ // Handle text/JSON messages (logs, framerate updates, frontend payloads, etc.)
else if (typeof event.data === 'string') {
// Skip heartbeat pong responses β€” they're plain text, not JSON
if (event.data === 'pong') return;
try {
const jsonData = JSON.parse(event.data);
- console.log(JSON.stringify(jsonData, null, 2));
+
// Handle log records
if (isLogRecord(jsonData)) {
logStoreRef.current.add(jsonData);
@@ -270,45 +275,30 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({childr
}
}
+ // Keypoints arrive as plain JSON objects β€” assign directly, no remapping.
if (jsonData.keypoints_raw) {
- const pointsMap = new Map<string, Point3d>();
- for (const [name, pt] of Object.entries(jsonData.keypoints_raw)) {
- const p = pt as { x: number; y: number; z: number };
- pointsMap.set(name, {x: p.x, y: p.y, z: p.z});
- }
- trackedPointsRef.current = pointsMap;
+ trackedPointsRef.current = jsonData.keypoints_raw as Record<string, Point3d>;
for (const cb of trackedPointsSubscribersRef.current) {
- cb(pointsMap);
+ cb(trackedPointsRef.current);
}
}
- }
+ if (jsonData.keypoints_filtered) {
+ keypointsFilteredRef.current = jsonData.keypoints_filtered as Record<string, Point3d>;
+ for (const cb of keypointsFilteredSubscribersRef.current) {
+ cb(keypointsFilteredRef.current);
+ }
+ }
- // // Handle tracked 3D points
- // else if (jsonData.tracked_points3d && typeof jsonData.tracked_points3d === 'object') {
- // const pointsMap = new Map<string, Point3d>();
- // for (const [name, pt] of Object.entries(jsonData.tracked_points3d)) {
- // const p = pt as { x: number; y: number; z: number };
- // pointsMap.set(name, {x: p.x, y: p.y, z: p.z});
- // }
- // trackedPointsRef.current = pointsMap;
- // for (const cb of trackedPointsSubscribersRef.current) {
- // cb(pointsMap);
- // }
- // }
- // // Handle rigid body poses
- // else if (jsonData.rigid_body_poses && typeof jsonData.rigid_body_poses === 'object') {
- // const posesMap = new Map<string, RigidBodyPose>();
- // for (const [key, pose] of Object.entries(jsonData.rigid_body_poses)) {
- // posesMap.set(key, pose as RigidBodyPose);
- // }
- // rigidBodiesRef.current = posesMap;
- // for (const cb of rigidBodiesSubscribersRef.current) {
- // cb(posesMap);
- // }
- // }
- // Handle other message types (silently ignored to avoid
- // retaining object references in the DevTools console)
+ if (jsonData.rigid_body_poses) {
+ const posesMap = new Map<string, RigidBodyPose>();
+ for (const [key, pose] of Object.entries(jsonData.rigid_body_poses)) {
+ posesMap.set(key, pose as RigidBodyPose);
+ }
+ rigidBodiesRef.current = posesMap;
+ for (const cb of rigidBodiesSubscribersRef.current) cb(posesMap);
+ }
+ }
} catch (error) {
console.error('Error parsing JSON message:', error);
}
@@ -364,11 +354,14 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({childr
return logStoreRef.current;
}, []);
- const subscribeToTrackedPoints = useCallback((cb: (points: Map<string, Point3d>) => void): () => void => {
+ const subscribeToKeypointsRaw = useCallback((cb: (points: Record<string, Point3d>) => void): () => void => {
trackedPointsSubscribersRef.current.add(cb);
- return () => {
- trackedPointsSubscribersRef.current.delete(cb);
- };
+ return () => { trackedPointsSubscribersRef.current.delete(cb); };
+ }, []);
+
+ const subscribeToKeypointsFiltered = useCallback((cb: (points: Record<string, Point3d>) => void): () => void => {
+ keypointsFilteredSubscribersRef.current.add(cb);
+ return () => { keypointsFilteredSubscribersRef.current.delete(cb); };
}, []);
const subscribeToRigidBodies = useCallback((cb: (poses: Map<string, RigidBodyPose>) => void): () => void => {
@@ -378,7 +371,7 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({childr
};
}, []);
- const getLatestTrackedPoints = useCallback((): Map<string, Point3d> => {
+ const getLatestKeypointsRaw = useCallback((): Record<string, Point3d> => {
return trackedPointsRef.current;
}, []);
@@ -408,10 +401,11 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({childr
getLogStore,
connectedCameraIds,
updateServerConnection,
- subscribeToTrackedPoints,
+ subscribeToKeypointsRaw,
+ subscribeToKeypointsFiltered,
subscribeToRigidBodies,
- getLatestTrackedPoints,
- }), [isConnected, connectedCameraIds, connect, disconnect, sendWebsocketMessage, setCanvasForCamera, getFps, getServerFps, getFramerateStore, getLogStore, updateServerConnection, subscribeToTrackedPoints, subscribeToRigidBodies, getLatestTrackedPoints]);
+ getLatestKeypointsRaw,
+ }), [isConnected, connectedCameraIds, connect, disconnect, sendWebsocketMessage, setCanvasForCamera, getFps, getServerFps, getFramerateStore, getLogStore, updateServerConnection, subscribeToKeypointsRaw, subscribeToKeypointsFiltered, subscribeToRigidBodies, getLatestKeypointsRaw]);
return (
<ServerContext.Provider value={contextValue}>
diff --git a/freemocap-ui/src/services/server/server-helpers/websocket-message-types.ts b/freemocap-ui/src/services/server/server-helpers/websocket-message-types.ts
index 5afca68f..4505566c 100644
--- a/freemocap-ui/src/services/server/server-helpers/websocket-message-types.ts
+++ b/freemocap-ui/src/services/server/server-helpers/websocket-message-types.ts
@@ -12,7 +12,7 @@ import {ModelInfo} from "@/services/server/server-helpers/image-overlay/image-ov
import {OverlayRendererFactory} from "@/services/server/server-helpers/image-overlay/overlay-renderer-factory";
import {DetailedFramerate} from "@/services/server/server-helpers/framerate-store";
import {LogRecord} from "@/services/server/server-helpers/log-store";
-import {Point3d, RigidBodyPose} from "@/components/viewport3d/viewport3d-types";
+import {Point3d, RigidBodyPose} from "@/components/viewport3d";
// Type guard to check if a message is a log record
export function isLogRecord(data: any): data is LogRecord {
diff --git a/freemocap/api/websocket/websocket_server.py b/freemocap/api/websocket/websocket_server.py
index 403479e3..6e924560 100644
--- a/freemocap/api/websocket/websocket_server.py
+++ b/freemocap/api/websocket/websocket_server.py
@@ -128,65 +128,39 @@ class WebsocketServer:
return self.last_received_frontend_confirmation >= self.last_sent_frame_number
async def _frontend_image_relay(self) -> None:
- """Relay image payloads from shared memory to the frontend via WebSocket."""
logger.info("Starting frontend image payload relay...")
try:
skipped_previous = False
while self.should_continue:
await await_10ms()
- if self.check_frame_acknowledgment_status():
- if skipped_previous:
- skipped_previous = False
- else:
- try:
- frontend_payloads = self._app.get_latest_frontend_payloads(
- if_newer_than=self.last_sent_frame_number
- )
- except IndexError:
- logger.warning(
- f"Ring buffer overwrite detected (last_sent_frame_number={self.last_sent_frame_number}). "
- f"Resetting to latest frame."
- )
- self.last_sent_frame_number = -1
- continue
+ if not self.check_frame_acknowledgment_status():
+ skipped_previous = True
+ backpressure = self.last_sent_frame_number - self.last_received_frontend_confirmation
+ if backpressure > BACKPRESSURE_WARNING_THRESHOLD and backpressure % BACKPRESSURE_WARNING_THRESHOLD == 0:
+ logger.trace(f"Backpressure: {backpressure} frames unacknowledged")
+ continue
- for pipeline_id, (image_payload_bytes, frontend_payload) in frontend_payloads.items():
- frame_number = None
- if not image_payload_bytes and not frontend_payload:
- continue
- if frontend_payload:
- if not isinstance(frontend_payload, FrontendPayload):
- frame_number = frontend_payload
- frontend_payload = None
- else:
- frame_number = frontend_payload.frame_number
- async with self._send_lock:
- await self.websocket.send_json(frontend_payload.model_dump())
+ if skipped_previous:
+ skipped_previous = False
+ continue
- if image_payload_bytes:
- if not isinstance(image_payload_bytes, (bytes, bytearray)):
- raise TypeError(
- f"Invalid payload bytes on frame {frame_number} - "
- f"got type {type(image_payload_bytes).__name__}"
- )
- async with self._send_lock:
- await self.websocket.send_bytes(image_payload_bytes)
+ try:
+ packets = self._app.get_latest_frontend_payloads(if_newer_than=self.last_sent_frame_number)
+ except IndexError:
+ logger.warning("Ring buffer overwrite β€” resetting to latest frame")
+ self.last_sent_frame_number = -1
+ continue
+ for packet in packets.values():
+ if packet.frontend_payload is not None:
+ async with self._send_lock:
+ await self.websocket.send_json(packet.frontend_payload.model_dump())
- if frame_number is not None:
- self.last_sent_frame_number = frame_number
- else:
- skipped_previous = True
- backpressure = self.last_sent_frame_number - self.last_received_frontend_confirmation
- if (
- backpressure > BACKPRESSURE_WARNING_THRESHOLD
- and backpressure % BACKPRESSURE_WARNING_THRESHOLD == 0
- ):
- logger.trace(
- f"Backpressure detected: {backpressure} frames not acknowledged by frontend! "
- f"Last sent frame: {self.last_sent_frame_number}, "
- f"last received confirmation: {self.last_received_frontend_confirmation}"
- )
+ if packet.image_bytes is not None:
+ async with self._send_lock:
+ await self.websocket.send_bytes(packet.image_bytes)
+
+ self.last_sent_frame_number = packet.frame_number
except WebSocketDisconnect:
logger.api("Client disconnected, ending Frontend Image relay task...")
diff --git a/freemocap/app/freemocap_application.py b/freemocap/app/freemocap_application.py
index 4091f731..10da7cae 100644
--- a/freemocap/app/freemocap_application.py
+++ b/freemocap/app/freemocap_application.py
@@ -21,15 +21,13 @@ from freemocap.core.pipeline.realtime.realtime_pipeline_manager import RealtimeP
from freemocap.core.tasks.calibration.calibration_task_config import PosthocCalibrationPipelineConfig
from freemocap.core.tasks.mocap.mocap_task_config import PosthocMocapPipelineConfig
from freemocap.core.types.type_overloads import PipelineIdString, FrameNumberInt
-from freemocap.core.viz.frontend_payload import FrontendPayload
+from freemocap.core.viz.frontend_payload import FrontendPayload, FrontendImagePacket
logger = logging.getLogger(__name__)
-
@dataclass
class FreemocapApplication:
-
global_kill_flag: multiprocessing.Value
worker_registry: WorkerRegistry
realtime_pipeline_manager: RealtimePipelineManager
@@ -65,8 +63,8 @@ class FreemocapApplication:
# ------------------------------------------------------------------
async def create_or_update_realtime_pipeline(
- self,
- pipeline_config: RealtimePipelineConfig,
+ self,
+ pipeline_config: RealtimePipelineConfig,
) -> RealtimePipeline:
existing = self.realtime_pipeline_manager.get_pipeline_by_camera_ids(
camera_ids=pipeline_config.camera_ids,
@@ -91,9 +89,9 @@ class FreemocapApplication:
# ------------------------------------------------------------------
async def create_posthoc_calibration_pipeline(
- self,
- recording_info: RecordingInfo,
- calibration_config: PosthocCalibrationPipelineConfig,
+ self,
+ recording_info: RecordingInfo,
+ calibration_config: PosthocCalibrationPipelineConfig,
) -> PosthocPipeline:
pipeline = self.posthoc_pipeline_manager.create_calibration_pipeline(
recording_info=recording_info,
@@ -103,9 +101,9 @@ class FreemocapApplication:
return pipeline
async def create_posthoc_mocap_pipeline(
- self,
- recording_info: RecordingInfo,
- mocap_config: PosthocMocapPipelineConfig,
+ self,
+ recording_info: RecordingInfo,
+ mocap_config: PosthocMocapPipelineConfig,
) -> PosthocPipeline:
pipeline = self.posthoc_pipeline_manager.create_mocap_pipeline(
recording_info=recording_info,
@@ -141,26 +139,31 @@ class FreemocapApplication:
# ------------------------------------------------------------------
def get_latest_frontend_payloads(
- self,
- if_newer_than: FrameNumberInt,
- ) -> dict[str, tuple[bytes, int]] | dict[str, tuple[bytes | None, FrontendPayload | None]]:
- # Clean up completed posthoc pipelines (releases relay threads + queues)
+ self,
+ if_newer_than: FrameNumberInt,
+ ) -> dict[str, FrontendImagePacket]:
self.posthoc_pipeline_manager.evict_completed()
realtime_pipelines = self.realtime_pipeline_manager.pipelines
- if len(realtime_pipelines) == 0 or all(
- not p.alive for p in realtime_pipelines.values()
- ):
- cg_payloads = self.camera_group_manager.get_latest_frontend_payloads(
- if_newer_than=if_newer_than,
- )
- return {
- camera_group_id: (payload[-1], payload[0])
- for camera_group_id, payload in cg_payloads.items()
- }
-
+ active_pipelines = [p for p in realtime_pipelines.values() if p.alive]
+
+ if not active_pipelines:
+ # Camera-only path
+ result = {}
+ for cg_id, payload in self.camera_group_manager.get_latest_frontend_payloads(
+ if_newer_than=if_newer_than
+ ).items():
+ frame_number, _timestamp, image_bytes = payload # unpack the known tuple shape
+ result[cg_id] = FrontendImagePacket(
+ image_bytes=image_bytes,
+ frame_number=frame_number,
+ frontend_payload=None,
+ )
+ return result
+
+ # Pipeline path β€” delegate to manager, which also returns FrontendImagePacket
return self.realtime_pipeline_manager.get_latest_frontend_payloads(
- if_newer_than=if_newer_than,
+ if_newer_than=if_newer_than
)
# ------------------------------------------------------------------
@@ -182,7 +185,6 @@ class FreemocapApplication:
self.posthoc_pipeline_manager.shutdown()
-
FREEMOCAP_APP: FreemocapApplication | None = None
diff --git a/freemocap/core/pipeline/realtime/realtime_pipeline.py b/freemocap/core/pipeline/realtime/realtime_pipeline.py
index c83c88b3..3afb353f 100644
--- a/freemocap/core/pipeline/realtime/realtime_pipeline.py
+++ b/freemocap/core/pipeline/realtime/realtime_pipeline.py
@@ -24,7 +24,7 @@ from freemocap.core.pipeline.realtime.camera_node import CameraNode
from freemocap.core.pipeline.realtime.realtime_aggregator_node import RealtimeAggregatorNode
from freemocap.core.pipeline.realtime.realtime_pipeline_config import RealtimePipelineConfig
from freemocap.core.types.type_overloads import PipelineIdString, TopicSubscriptionQueue, FrameNumberInt
-from freemocap.core.viz.frontend_payload import FrontendPayload
+from freemocap.core.viz.frontend_payload import FrontendPayload, FrontendImagePacket
from freemocap.pubsub.pubsub_manager import PubSubTopicManager
from freemocap.pubsub.pubsub_topics import (
AggregationNodeOutputTopic,
@@ -62,8 +62,8 @@ class RealtimePipeline:
if not self.started:
return False
return (
- all(node.is_alive for node in self.camera_nodes.values())
- and self.aggregation_node.is_alive
+ all(node.is_alive for node in self.camera_nodes.values())
+ and self.aggregation_node.is_alive
)
@property
@@ -80,11 +80,11 @@ class RealtimePipeline:
@classmethod
def create(
- cls,
- *,
- camera_group: CameraGroup,
- worker_registry: WorkerRegistry,
- pipeline_config: RealtimePipelineConfig,
+ cls,
+ *,
+ camera_group: CameraGroup,
+ worker_registry: WorkerRegistry,
+ pipeline_config: RealtimePipelineConfig,
) -> "RealtimePipeline":
global_kill_flag = camera_group.ipc.global_kill_flag
@@ -185,21 +185,17 @@ class RealtimePipeline:
requested_configs=camera_configs,
)
- def get_latest_frontend_payload(
- self,
- if_newer_than: FrameNumberInt,
- ) -> None | tuple[Any, None] | tuple[bytes | None, FrontendPayload]:
- """
- Drain the aggregation output queue and return the latest payload.
- Returns None if no new data is available.
- """
+ def get_latest_frontend_payload(self, if_newer_than: FrameNumberInt, ) -> FrontendImagePacket | None:
if not self.alive:
if self.camera_group.alive:
- _, _, frames_bytearray = self.camera_group.get_latest_frontend_payload(
- if_newer_than=if_newer_than,
- )
- if frames_bytearray is not None:
- return (frames_bytearray, None)
+ result = self.camera_group.get_latest_frontend_payload(if_newer_than=if_newer_than)
+ if result is not None:
+ frame_number, _ts, frames_bytes = result
+ return FrontendImagePacket(
+ image_bytes=frames_bytes,
+ frame_number=frame_number,
+ frontend_payload=None,
+ )
return None
aggregation_output: AggregationNodeOutputMessage | None = None
@@ -209,11 +205,26 @@ class RealtimePipeline:
if aggregation_output is None:
return None
- frames_bytearray = self.camera_group.get_frontend_payload_by_frame_number(
+ frames_bytes = self.camera_group.get_frontend_payload_by_frame_number(
+ frame_number=aggregation_output.frame_number,
+ )
+ return FrontendImagePacket(
+ image_bytes=frames_bytes,
+ frame_number=aggregation_output.frame_number,
+ frontend_payload=FrontendPayload.from_aggregation_output(aggregation_output),
+ )
+ aggregation_output: AggregationNodeOutputMessage | None = None
+ while not self.aggregation_output_subscription.empty():
+ aggregation_output = self.aggregation_output_subscription.get()
+
+ if aggregation_output is None:
+ return None, None
+
+ frames_bytes = self.camera_group.get_frontend_payload_by_frame_number(
frame_number=aggregation_output.frame_number,
)
return (
- frames_bytearray,
+ frames_bytes,
FrontendPayload.from_aggregation_output(aggregation_output=aggregation_output),
)
diff --git a/freemocap/core/pipeline/realtime/realtime_pipeline_manager.py b/freemocap/core/pipeline/realtime/realtime_pipeline_manager.py
index 2f214c94..faac6a8f 100644
--- a/freemocap/core/pipeline/realtime/realtime_pipeline_manager.py
+++ b/freemocap/core/pipeline/realtime/realtime_pipeline_manager.py
@@ -23,7 +23,7 @@ from freemocap.core.pipeline.abcs.pipeline_manager_abc import PipelineManagerABC
from freemocap.core.pipeline.realtime.realtime_aggregator_node import RealtimePipelineConfig
from freemocap.core.pipeline.realtime.realtime_pipeline import RealtimePipeline
from freemocap.core.types.type_overloads import PipelineIdString, FrameNumberInt
-from freemocap.core.viz.frontend_payload import FrontendPayload
+from freemocap.core.viz.frontend_payload import FrontendPayload, FrontendImagePacket
logger = logging.getLogger(__name__)
@@ -104,15 +104,15 @@ class RealtimePipelineManager(PipelineManagerABC):
# ------------------------------------------------------------------
def get_latest_frontend_payloads(
- self,
- if_newer_than: FrameNumberInt,
- ) -> dict[PipelineIdString, tuple[bytes | None, FrontendPayload | None]]:
- latest: dict[PipelineIdString, tuple[bytes | None, FrontendPayload | None]] = {}
+ self,
+ if_newer_than: FrameNumberInt,
+ ) -> dict[PipelineIdString, FrontendImagePacket]:
+ latest: dict[PipelineIdString, FrontendImagePacket] = {}
with self.lock:
for pipeline_id, pipeline in self.pipelines.items():
- output = pipeline.get_latest_frontend_payload(if_newer_than=if_newer_than)
- if output is not None:
- latest[pipeline_id] = output
+ packet = pipeline.get_latest_frontend_payload(if_newer_than=if_newer_than)
+ if packet is not None:
+ latest[pipeline_id] = packet
return latest
# ------------------------------------------------------------------
diff --git a/freemocap/core/viz/frontend_payload.py b/freemocap/core/viz/frontend_payload.py
index 6cfbbdce..966ec304 100644
--- a/freemocap/core/viz/frontend_payload.py
+++ b/freemocap/core/viz/frontend_payload.py
@@ -12,6 +12,11 @@ from freemocap.pubsub.pubsub_topics import AggregationNodeOutputMessage
logger = logging.getLogger(__name__)
+# freemocap/core/viz/frontend_image_packet.py
+from dataclasses import dataclass
+
+
+
class FrontendPayload(BaseModel):
"""Complete payload for frontend visualization.
@@ -19,17 +24,14 @@ class FrontendPayload(BaseModel):
Carries both charuco and mediapipe overlay data separately so both
can be active simultaneously per camera.
"""
- model_config = ConfigDict(
- arbitrary_types_allowed=True,
- validate_assignment=False
- )
- message_type: str = "frontend_payload"
+
frame_number: int
charuco_overlays: dict[CameraIdString, CharucoOverlayData]
skeleton_overlays: dict[CameraIdString, MediapipeOverlayData]
keypoints_raw: dict[TrackedPointNameString, Point3d]
keypoints_filtered: dict[TrackedPointNameString, Point3d]
rigid_body_poses: dict[str, RigidBodyPose]
+ message_type: str = "frontend_payload"
@classmethod
def from_aggregation_output(
@@ -45,3 +47,16 @@ class FrontendPayload(BaseModel):
keypoints_filtered=aggregation_output.keypoints_filtered,
rigid_body_poses=aggregation_output.rigid_body_poses,
)
+
+@dataclass(slots=True, frozen=True)
+class FrontendImagePacket:
+ """
+ Unified output type for all frontend image relay paths.
+
+ `frontend_payload` is None in camera-only mode (no active pipeline).
+ `frame_number` is always populated so the relay can track acknowledgment
+ without any isinstance checks.
+ """
+ image_bytes: bytes | None
+ frame_number: int
+ frontend_payload: FrontendPayload | None # None = camera-only mode