Free Motion Capture for Everyone πβ¨
TypeScript
gettin there
jonmatthiscb148bb82842
diff --git a/freemocap-ui/src/components/viewport3d/RigidBodyRenderer.tsx b/freemocap-ui/src/components/old-viewport3d/RigidBodyRenderer.tsxsimilarity index 98%rename from freemocap-ui/src/components/viewport3d/RigidBodyRenderer.tsxrename to freemocap-ui/src/components/old-viewport3d/RigidBodyRenderer.tsxindex d5f4a99f..bcea9bf6 100644--- a/freemocap-ui/src/components/viewport3d/RigidBodyRenderer.tsx+++ b/freemocap-ui/src/components/old-viewport3d/RigidBodyRenderer.tsx@@ -2,7 +2,7 @@ import {useEffect, useMemo, useRef} from "react";import {Color, CylinderGeometry, InstancedMesh, MeshStandardMaterial, Object3D, Vector3,} from "three";import {useFrame} from "@react-three/fiber";import {useServer} from "@/services";-import {RigidBodyPose} from "@/components/viewport3d/viewport3d-types";+import {RigidBodyPose} from "@/components/viewport3d/helpers/viewport3d-types";import {Z_OFFSET} from "@/components/viewport3d/skeleton-config";import {SKELETON_COLORS} from "@/components/viewport3d/skeleton-colors";diff --git a/freemocap-ui/src/components/viewport3d/SceneEnvironment.tsx b/freemocap-ui/src/components/old-viewport3d/SceneEnvironment.tsxsimilarity index 100%rename from freemocap-ui/src/components/viewport3d/SceneEnvironment.tsxrename to freemocap-ui/src/components/old-viewport3d/SceneEnvironment.tsxdiff --git a/freemocap-ui/src/components/viewport3d/SkeletonRenderer.tsx b/freemocap-ui/src/components/old-viewport3d/SkeletonRenderer.tsxsimilarity index 99%rename from freemocap-ui/src/components/viewport3d/SkeletonRenderer.tsxrename to freemocap-ui/src/components/old-viewport3d/SkeletonRenderer.tsxindex 59257a26..f7270691 100644--- a/freemocap-ui/src/components/viewport3d/SkeletonRenderer.tsx+++ b/freemocap-ui/src/components/old-viewport3d/SkeletonRenderer.tsx@@ -12,7 +12,7 @@ import {} from "three";import {useFrame} from "@react-three/fiber";import {useServer} from "@/services";-import {Point3d, PointStyle} from "@/components/viewport3d/viewport3d-types";+import {Point3d, PointStyle} from "@/components/viewport3d/helpers/viewport3d-types";import {getPointStyle,MAX_POINTS,diff --git a/freemocap-ui/src/components/old-viewport3d/ThreeJsCanvas.tsx b/freemocap-ui/src/components/old-viewport3d/ThreeJsCanvas.tsxnew file mode 100644index 00000000..fc10ae91--- /dev/null+++ b/freemocap-ui/src/components/old-viewport3d/ThreeJsCanvas.tsx@@ -0,0 +1,100 @@+import {useCallback, useEffect, useRef} from "react";+import {Canvas} from "@react-three/fiber";+import {Box, IconButton, Tooltip, Typography} from "@mui/material";+import CenterFocusStrongIcon from "@mui/icons-material/CenterFocusStrong";+import HomeIcon from "@mui/icons-material/Home";+import type CameraControlsImpl from "camera-controls";+import {ThreeJsScene} from "@/components/viewport3d/ThreeJsScene";+import {fitCameraToSkeleton} from "@/components/viewport3d/fit-camera";+import {useServer} from "@/services";++export function ThreeJsCanvas() {+ const cameraControlsRef = useRef<CameraControlsImpl>(null!);+ const containerRef = useRef<HTMLDivElement>(null);+ const { getLatestTrackedPoints } = useServer();++ const handleFitToSkeleton = useCallback(() => {+ fitCameraToSkeleton(cameraControlsRef.current, getLatestTrackedPoints());+ }, [getLatestTrackedPoints]);++ // Press "F" while the viewport container is focused to fit camera to skeleton+ useEffect(() => {+ const container = containerRef.current;+ if (!container) return;++ const handleKeyDown = (e: KeyboardEvent) => {+ if (e.key === "f" || e.key === "F") {+ e.preventDefault();+ handleFitToSkeleton();+ }+ };++ container.addEventListener("keydown", handleKeyDown);+ return () => container.removeEventListener("keydown", handleKeyDown);+ }, [handleFitToSkeleton]);++ return (+ <Box+ ref={containerRef}+ tabIndex={0}+ sx={{+ width: '100%',+ height: '100%',+ position: 'relative',+ outline: 'none',+ }}+ >+ <Canvas+ shadows+ camera={{ position: [1e3, 1e3, 1e3], fov: 75, near: 0.1, far: 10000 }}+ gl={{ antialias: true, logarithmicDepthBuffer: true }}+ >+ <ThreeJsScene cameraControlsRef={cameraControlsRef} />+ </Canvas>++ <Box sx={{ position: 'absolute', left: 16, bottom: 16, bgcolor: 'background.paper', p: 1, borderRadius: 1, boxShadow: 1 }}>+ <Typography variant="caption" color="text.secondary">+ Rotate: drag β’ Zoom: scroll β’ Pan: right-drag or two-finger+ </Typography>+ </Box>++ <Box sx={{ position: 'absolute', bottom: 16, right: 16, display: 'flex', gap: 1 }}>+ <Tooltip title="Fit camera to skeleton (F)">+ <IconButton+ onClick={handleFitToSkeleton}+ size="small"+ sx={{+ bgcolor: 'background.paper',+ boxShadow: 2,+ '&:hover': { bgcolor: 'action.hover' },+ }}+ >+ <CenterFocusStrongIcon fontSize="small" />+ </IconButton>+ </Tooltip>++ <Tooltip title="Reset view">+ <IconButton+ onClick={() => {+ const cam = cameraControlsRef.current?.camera;+ if (cam) {+ cam.position.set(1e3, 1e3, 1e3);+ cam.lookAt(0, 0, 0);+ cam.updateProjectionMatrix();+ }+ cameraControlsRef.current?.reset(true);+ }}+ size="small"+ sx={{+ bgcolor: 'background.paper',+ boxShadow: 2,+ '&:hover': { bgcolor: 'action.hover' },+ }}+ >+ <HomeIcon fontSize="small" />+ </IconButton>+ </Tooltip>+ </Box>+ </Box>+ );+}diff --git a/freemocap-ui/src/components/old-viewport3d/ThreeJsScene.tsx b/freemocap-ui/src/components/old-viewport3d/ThreeJsScene.tsxnew file mode 100644index 00000000..e1d51668--- /dev/null+++ b/freemocap-ui/src/components/old-viewport3d/ThreeJsScene.tsx@@ -0,0 +1,22 @@+import {RefObject} from "react";+import {CameraControls} from "@react-three/drei";+import type CameraControlsImpl from "camera-controls";+import {SceneEnvironment} from "@/components/viewport3d/scene/SceneEnvironment";+import {SkeletonRenderer} from "@/components/viewport3d/SkeletonRenderer";+import {RigidBodyRenderer} from "@/components/viewport3d/renderers/RigidBodyRenderer";++interface ThreeJsSceneProps {+ cameraControlsRef: RefObject<CameraControlsImpl>;+}++/** Top-level scene graph: camera controls, environment, and skeleton visualization. */+export function ThreeJsScene({ cameraControlsRef }: ThreeJsSceneProps) {+ return (+ <>+ <CameraControls ref={cameraControlsRef} makeDefault />+ <SceneEnvironment />+ <SkeletonRenderer />+ {/*<RigidBodyRenderer />*/}+ </>+ );+}diff --git a/freemocap-ui/src/components/viewport3d/face-contours.ts b/freemocap-ui/src/components/old-viewport3d/face-contours.tssimilarity index 100%rename from freemocap-ui/src/components/viewport3d/face-contours.tsrename to freemocap-ui/src/components/old-viewport3d/face-contours.tsdiff --git a/freemocap-ui/src/components/viewport3d/fit-camera.ts b/freemocap-ui/src/components/old-viewport3d/fit-camera.tssimilarity index 100%rename from freemocap-ui/src/components/viewport3d/fit-camera.tsrename to freemocap-ui/src/components/old-viewport3d/fit-camera.tsdiff --git a/freemocap-ui/src/components/viewport3d/skeleton-colors.ts b/freemocap-ui/src/components/old-viewport3d/skeleton-colors.tssimilarity index 100%rename from freemocap-ui/src/components/viewport3d/skeleton-colors.tsrename to freemocap-ui/src/components/old-viewport3d/skeleton-colors.tsdiff --git a/freemocap-ui/src/components/viewport3d/skeleton-config.ts b/freemocap-ui/src/components/old-viewport3d/skeleton-config.tssimilarity index 98%rename from freemocap-ui/src/components/viewport3d/skeleton-config.tsrename to freemocap-ui/src/components/old-viewport3d/skeleton-config.tsindex 256e688f..4ba8d981 100644--- a/freemocap-ui/src/components/viewport3d/skeleton-config.ts+++ b/freemocap-ui/src/components/old-viewport3d/skeleton-config.ts@@ -1,6 +1,6 @@import {Color} from "three";-import {PointStyle} from "@/components/viewport3d/viewport3d-types";-import {buildFaceContourColors, buildFaceContourSegments} from "@/components/viewport3d/face-contours";+import {PointStyle} from "@/components/viewport3d/helpers/viewport3d-types";+import {buildFaceContourColors, buildFaceContourSegments} from "@/components/viewport3d/helpers/face-contours";import {SKELETON_COLORS} from "@/components/viewport3d/skeleton-colors";// Maximum tracked points the instanced mesh can holddiff --git a/freemocap-ui/src/components/viewport3d/viewport3d-types.ts b/freemocap-ui/src/components/old-viewport3d/viewport3d-types.tssimilarity index 100%rename from freemocap-ui/src/components/viewport3d/viewport3d-types.tsrename to freemocap-ui/src/components/old-viewport3d/viewport3d-types.tsdiff --git a/freemocap-ui/src/components/viewport3d/virtual-points.ts b/freemocap-ui/src/components/old-viewport3d/virtual-points.tssimilarity index 94%rename from freemocap-ui/src/components/viewport3d/virtual-points.tsrename to freemocap-ui/src/components/old-viewport3d/virtual-points.tsindex 1daf859f..85c8348b 100644--- a/freemocap-ui/src/components/viewport3d/virtual-points.ts+++ b/freemocap-ui/src/components/old-viewport3d/virtual-points.ts@@ -1,4 +1,4 @@-import {Point3d} from "@/components/viewport3d/viewport3d-types";+import {Point3d} from "@/components/viewport3d/helpers/viewport3d-types";/*** Resolve a point name to a 3D position. Handles both raw landmark namesdiff --git a/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsx b/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsxindex fc10ae91..c7a40e44 100644--- a/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsx+++ b/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsx@@ -1,100 +1,54 @@-import {useCallback, useEffect, useRef} from "react";-import {Canvas} from "@react-three/fiber";-import {Box, IconButton, Tooltip, Typography} from "@mui/material";-import CenterFocusStrongIcon from "@mui/icons-material/CenterFocusStrong";-import HomeIcon from "@mui/icons-material/Home";+import { useCallback, useEffect, useRef } from "react";+import { Canvas } from "@react-three/fiber";+import { Box } from "@mui/material";import type CameraControlsImpl from "camera-controls";-import {ThreeJsScene} from "@/components/viewport3d/ThreeJsScene";-import {fitCameraToSkeleton} from "@/components/viewport3d/fit-camera";-import {useServer} from "@/services";+import { ThreeJsScene } from "./ThreeJsScene";+import { ViewportOverlay } from "./scene/ViewportOverlay";+import { ViewportStateProvider } from "./scene/ViewportStateContext";+import { fitCameraToPoints } from "./scene/SceneCamera";+import { useServer } from "@/services";export function ThreeJsCanvas() {- const cameraControlsRef = useRef<CameraControlsImpl>(null!);+ const controlsRef = useRef<CameraControlsImpl>(null!);const containerRef = useRef<HTMLDivElement>(null);- const { getLatestTrackedPoints } = useServer();+ const { getLatestKeypointsRaw } = useServer();- const handleFitToSkeleton = useCallback(() => {- fitCameraToSkeleton(cameraControlsRef.current, getLatestTrackedPoints());- }, [getLatestTrackedPoints]);+ const handleFit = useCallback(() => {+ fitCameraToPoints(controlsRef.current, getLatestKeypointsRaw());+ }, [getLatestKeypointsRaw]);- // Press "F" while the viewport container is focused to fit camera to skeleton- useEffect(() => {- const container = containerRef.current;- if (!container) return;+ const handleReset = useCallback(() => {+ controlsRef.current?.reset(true);+ }, []);- const handleKeyDown = (e: KeyboardEvent) => {- if (e.key === "f" || e.key === "F") {- e.preventDefault();- handleFitToSkeleton();- }+ // "F" key shortcut+ useEffect(() => {+ const el = containerRef.current;+ if (!el) return;+ const onKey = (e: KeyboardEvent) => {+ if (e.key === "f" || e.key === "F") { e.preventDefault(); handleFit(); }};-- container.addEventListener("keydown", handleKeyDown);- return () => container.removeEventListener("keydown", handleKeyDown);- }, [handleFitToSkeleton]);+ el.addEventListener("keydown", onKey);+ return () => el.removeEventListener("keydown", onKey);+ }, [handleFit]);return (- <Box- ref={containerRef}- tabIndex={0}- sx={{- width: '100%',- height: '100%',- position: 'relative',- outline: 'none',- }}- >- <Canvas- shadows- camera={{ position: [1e3, 1e3, 1e3], fov: 75, near: 0.1, far: 10000 }}- gl={{ antialias: true, logarithmicDepthBuffer: true }}+ <ViewportStateProvider>+ <Box+ ref={containerRef}+ tabIndex={0}+ sx={{ width: "100%", height: "100%", position: "relative", outline: "none" }}>- <ThreeJsScene cameraControlsRef={cameraControlsRef} />- </Canvas>-- <Box sx={{ position: 'absolute', left: 16, bottom: 16, bgcolor: 'background.paper', p: 1, borderRadius: 1, boxShadow: 1 }}>- <Typography variant="caption" color="text.secondary">- Rotate: drag β’ Zoom: scroll β’ Pan: right-drag or two-finger- </Typography>- </Box>-- <Box sx={{ position: 'absolute', bottom: 16, right: 16, display: 'flex', gap: 1 }}>- <Tooltip title="Fit camera to skeleton (F)">- <IconButton- onClick={handleFitToSkeleton}- size="small"- sx={{- bgcolor: 'background.paper',- boxShadow: 2,- '&:hover': { bgcolor: 'action.hover' },- }}- >- <CenterFocusStrongIcon fontSize="small" />- </IconButton>- </Tooltip>-- <Tooltip title="Reset view">- <IconButton- onClick={() => {- const cam = cameraControlsRef.current?.camera;- if (cam) {- cam.position.set(1e3, 1e3, 1e3);- cam.lookAt(0, 0, 0);- cam.updateProjectionMatrix();- }- cameraControlsRef.current?.reset(true);- }}- size="small"- sx={{- bgcolor: 'background.paper',- boxShadow: 2,- '&:hover': { bgcolor: 'action.hover' },- }}- >- <HomeIcon fontSize="small" />- </IconButton>- </Tooltip>+ <Canvas+ shadows+ camera={{ position: [1e3, 1e3, 1e3], fov: 75, near: 0.1, far: 1e5 }}+ gl={{ antialias: true, logarithmicDepthBuffer: true }}+ >+ <ThreeJsScene cameraControlsRef={controlsRef} />+ </Canvas>++ <ViewportOverlay onFitCamera={handleFit} onResetCamera={handleReset} /></Box>- </Box>+ </ViewportStateProvider>);}diff --git a/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsx b/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsxindex e207bca7..576b1de8 100644--- a/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsx+++ b/freemocap-ui/src/components/viewport3d/ThreeJsScene.tsx@@ -1,22 +1,23 @@import {RefObject} from "react";-import {CameraControls} from "@react-three/drei";import type CameraControlsImpl from "camera-controls";-import {SceneEnvironment} from "@/components/viewport3d/SceneEnvironment";-import {SkeletonRenderer} from "@/components/viewport3d/SkeletonRenderer";-import {RigidBodyRenderer} from "@/components/viewport3d/RigidBodyRenderer";+import {SceneCamera} from "./scene/SceneCamera";+import {SceneEnvironment} from "./scene/SceneEnvironment";+import {KeypointsRenderer} from "./renderers/KeypointsRenderer";interface ThreeJsSceneProps {cameraControlsRef: RefObject<CameraControlsImpl>;}-/** Top-level scene graph: camera controls, environment, and skeleton visualization. */export function ThreeJsScene({ cameraControlsRef }: ThreeJsSceneProps) {+ // const { visibility } = useViewportState();+return (<>- <CameraControls ref={cameraControlsRef} makeDefault />+ <SceneCamera controlsRef={cameraControlsRef} /><SceneEnvironment />- <SkeletonRenderer />- {/*<RigidBodyRenderer />*/}+ <KeypointsRenderer />+ {/*{visibility.rigidBodies && <RigidBodyRenderer />}*/}+ {/*{visibility.face && <FaceRenderer />}*/}</>);}diff --git a/freemocap-ui/src/components/viewport3d/helpers/colors.ts b/freemocap-ui/src/components/viewport3d/helpers/colors.tsnew file mode 100644index 00000000..12b2c05e--- /dev/null+++ b/freemocap-ui/src/components/viewport3d/helpers/colors.ts@@ -0,0 +1,15 @@+import { Color } from "three";++export const COLORS = {+ // Keypoints - raw are dimmer/smaller, filtered are brighter/larger+ raw: new Color("#6688AA"),+ filtered: new Color("#44FF88"),+ // Rigid bodies+ rigidBody: new Color("#CC8833"),+ rigidBodyX: new Color("#FF4444"),+ rigidBodyY: new Color("#44CC44"),+ // Face+ face: new Color("#FFD700"),+ // Utility+ hidden: new Color("#000000"),+} as const;diff --git a/freemocap-ui/src/components/viewport3d/helpers/face-contours.ts b/freemocap-ui/src/components/viewport3d/helpers/face-contours.tsnew file mode 100644index 00000000..f11481cc--- /dev/null+++ b/freemocap-ui/src/components/viewport3d/helpers/face-contours.ts@@ -0,0 +1,78 @@+/**+ * MediaPipe face mesh contour connections.+ * Point names follow `face.{group}_{mediapipe_index}`.+ */++interface FaceContourGroup {+ prefix: string;+ connections: [number, number][];+}++const FACE_CONTOUR_GROUPS: FaceContourGroup[] = [+ {+ prefix: "face.lips",+ connections: [+ [61,146],[146,91],[91,181],[181,84],[84,17],[17,314],[314,405],[405,321],[321,375],[375,291],+ [61,185],[185,40],[40,39],[39,37],[37,0],[0,267],[267,269],[269,270],[270,409],[409,291],+ [78,95],[95,88],[88,178],[178,87],[87,14],[14,317],[317,402],[402,318],[318,324],[324,308],+ [78,191],[191,80],[80,81],[81,82],[82,13],[13,312],[312,311],[311,310],[310,415],[415,308],+ ],+ },+ {+ prefix: "face.left_eye",+ connections: [+ [263,249],[249,390],[390,373],[373,374],[374,380],[380,381],[381,382],[382,362],+ [263,466],[466,388],[388,387],[387,386],[386,385],[385,384],[384,398],[398,362],+ ],+ },+ {+ prefix: "face.right_eye",+ connections: [+ [33,7],[7,163],[163,144],[144,145],[145,153],[153,154],[154,155],[155,133],+ [33,246],[246,161],[161,160],[160,159],[159,158],[158,157],[157,173],[173,133],+ ],+ },+ {+ prefix: "face.left_eyebrow",+ connections: [[276,283],[283,282],[282,295],[295,285],[300,293],[293,334],[334,296],[296,336]],+ },+ {+ prefix: "face.right_eyebrow",+ connections: [[46,53],[53,52],[52,65],[65,55],[70,63],[63,105],[105,66],[66,107]],+ },+ {+ prefix: "face.face_oval",+ connections: [+ [10,338],[338,297],[297,332],[332,284],[284,251],[251,389],[389,356],[356,454],+ [454,323],[323,361],[361,288],[288,397],[397,365],[365,379],[379,378],[378,400],+ [400,377],[377,152],[152,148],[148,176],[176,149],[149,150],[150,136],[136,172],+ [172,58],[58,132],[132,93],[93,234],[234,127],[127,162],[162,21],[21,54],[54,103],+ [103,67],[67,109],[109,10],+ ],+ },+ {+ prefix: "face.left_iris",+ connections: [[474,475],[475,476],[476,477],[477,474]],+ },+ {+ prefix: "face.right_iris",+ connections: [[469,470],[470,471],[471,472],[472,469]],+ },+];++export interface FaceSegment {+ from: string;+ to: string;+}++/** All face contour segments as pairs of point names. */+export const FACE_SEGMENTS: FaceSegment[] = [];++for (const group of FACE_CONTOUR_GROUPS) {+ for (const [a, b] of group.connections) {+ FACE_SEGMENTS.push({+ from: `${group.prefix}_${a}`,+ to: `${group.prefix}_${b}`,+ });+ }+}diff --git a/freemocap-ui/src/components/viewport3d/helpers/viewport3d-types.ts b/freemocap-ui/src/components/viewport3d/helpers/viewport3d-types.tsnew file mode 100644index 00000000..78bbe957--- /dev/null+++ b/freemocap-ui/src/components/viewport3d/helpers/viewport3d-types.ts@@ -0,0 +1,41 @@+import { Color } from "three";++/** 3D point */+export interface Point3d {+ x: number;+ y: number;+ z: number;+}++/** Rigid body pose from Python backend. */+export interface RigidBodyPose {+ bone_key: string;+ position: [number, number, number];+ orientation: [number, number, number, number]; // (w, x, y, z)+ scale: [number, number, number]; // (sx, sy, sz)+}++/** Viewport layer visibility toggles. */+export interface ViewportVisibility {+ environment: boolean;+ keypointsRaw: boolean;+ keypointsFiltered: boolean;+ rigidBodies: boolean;+ face: boolean;+}++export const DEFAULT_VISIBILITY: ViewportVisibility = {+ environment: true,+ keypointsRaw: true,+ keypointsFiltered: true,+ rigidBodies: true,+ face: true,+};++/** Live stats from each renderer. */+export interface ViewportStats {+ keypointsRaw: number;+ keypointsFiltered: number;+ rigidBodies: number;+ facePoints: number;+}diff --git a/freemocap-ui/src/components/viewport3d/index.ts b/freemocap-ui/src/components/viewport3d/index.tsnew file mode 100644index 00000000..981d348e--- /dev/null+++ b/freemocap-ui/src/components/viewport3d/index.ts@@ -0,0 +1,2 @@+export { ThreeJsCanvas } from "./ThreeJsCanvas";+export type { Point3d, RigidBodyPose, ViewportVisibility } from "./helpers/viewport3d-types";diff --git a/freemocap-ui/src/components/viewport3d/renderers/FaceRenderer.tsx b/freemocap-ui/src/components/viewport3d/renderers/FaceRenderer.tsxnew file mode 100644index 00000000..ce6fafb0--- /dev/null+++ b/freemocap-ui/src/components/viewport3d/renderers/FaceRenderer.tsx@@ -0,0 +1,136 @@+import { useEffect, useMemo, useRef } from "react";+import {+ BufferAttribute,+ BufferGeometry,+ LineBasicMaterial,+ LineSegments,+ InstancedMesh,+ MeshBasicMaterial,+ Object3D,+ SphereGeometry,+ Vector3,+} from "three";+import { useFrame } from "@react-three/fiber";+import { useServer } from "@/services";+import { Point3d } from "../helpers/viewport3d-types";+import { useViewportState } from "../scene/ViewportStateContext";+import { COLORS } from "../helpers/colors";+import { FACE_SEGMENTS } from "../helpers/face-contours";++const MAX_FACE_POINTS = 512;+const DOT_RADIUS = 0.015;+const DUMMY = new Object3D();+const FAR_AWAY = new Vector3(1e5, 1e5, 1e5);++/**+ * Renders face mesh as small dots + contour line segments.+ * Subscribes to raw keypoints and filters for face.* names.+ */+export function FaceRenderer() {+ const { subscribeToKeypointsRaw } = useServer();+ const { statsRef } = useViewportState();++ const dotsRef = useRef<InstancedMesh>(null);+ const linesRef = useRef<LineSegments>(null);++ const pointsRef = useRef<Map<string, Point3d>>(new Map());+ const nameToIdx = useRef<Map<string, number>>(new Map());+ const nextIdx = useRef(0);+ const dirtyRef = useRef(false);++ const dotGeo = useMemo(() => new SphereGeometry(1, 4, 3), []);+ const dotMat = useMemo(() => new MeshBasicMaterial({ color: COLORS.face }), []);++ const lineMat = useMemo(() => new LineBasicMaterial({+ color: COLORS.face,+ transparent: true,+ opacity: 0.7,+ }), []);++ const lineGeo = useMemo(() => {+ const g = new BufferGeometry();+ const n = FACE_SEGMENTS.length * 2;+ const pos = new Float32Array(n * 3).fill(1e5);+ g.setAttribute("position", new BufferAttribute(pos, 3));+ return g;+ }, []);++ useEffect(() => {+ return subscribeToKeypointsRaw((allPts: Map<string, Point3d>) => {+ const face = new Map<string, Point3d>();+ for (const [name, pt] of allPts) {+ if (name.startsWith("face.")) face.set(name, pt);+ }+ pointsRef.current = face;+ dirtyRef.current = true;++ for (const name of face.keys()) {+ if (!nameToIdx.current.has(name) && nextIdx.current < MAX_FACE_POINTS) {+ nameToIdx.current.set(name, nextIdx.current++);+ }+ }+ });+ }, [subscribeToKeypointsRaw]);++ useEffect(() => {+ const mesh = dotsRef.current;+ if (!mesh) return;+ for (let i = 0; i < MAX_FACE_POINTS; i++) {+ DUMMY.position.copy(FAR_AWAY);+ DUMMY.scale.set(0, 0, 0);+ DUMMY.updateMatrix();+ mesh.setMatrixAt(i, DUMMY.matrix);+ }+ mesh.instanceMatrix.needsUpdate = true;+ mesh.count = MAX_FACE_POINTS;+ }, []);++ useFrame(() => {+ const dots = dotsRef.current;+ if (!dots || !dirtyRef.current) return;++ const pts = pointsRef.current;+ let count = 0;++ for (const [name, idx] of nameToIdx.current) {+ const pt = pts.get(name);+ if (pt) {+ DUMMY.position.set(pt.x, pt.y, pt.z);+ DUMMY.scale.setScalar(DOT_RADIUS);+ count++;+ } else {+ DUMMY.position.copy(FAR_AWAY);+ DUMMY.scale.set(0, 0, 0);+ }+ DUMMY.updateMatrix();+ dots.setMatrixAt(idx, DUMMY.matrix);+ }+ dots.instanceMatrix.needsUpdate = true;++ // Update line segments+ const positions = lineGeo.attributes.position.array as Float32Array;+ for (let i = 0; i < FACE_SEGMENTS.length; i++) {+ const seg = FACE_SEGMENTS[i];+ const a = pts.get(seg.from);+ const b = pts.get(seg.to);+ const base = i * 6;+ if (a && b) {+ positions[base] = a.x; positions[base+1] = a.y; positions[base+2] = a.z;+ positions[base+3] = b.x; positions[base+4] = b.y; positions[base+5] = b.z;+ } else {+ for (let j = 0; j < 6; j++) positions[base + j] = 1e5;+ }+ }+ lineGeo.attributes.position.needsUpdate = true;++ dirtyRef.current = false;+ statsRef.current.facePoints = count;+ });++ return (+ <>+ <instancedMesh ref={dotsRef} args={[dotGeo, dotMat, MAX_FACE_POINTS]} frustumCulled={false} />+ <lineSegments ref={linesRef} geometry={lineGeo} material={lineMat} frustumCulled={false} />+ </>+ );+}diff --git a/freemocap-ui/src/components/viewport3d/renderers/KeypointsRenderer.tsx b/freemocap-ui/src/components/viewport3d/renderers/KeypointsRenderer.tsxnew file mode 100644index 00000000..34a6b48c--- /dev/null+++ b/freemocap-ui/src/components/viewport3d/renderers/KeypointsRenderer.tsx@@ -0,0 +1,131 @@+import { useEffect, useMemo, useRef } from "react";+import {+ Color,+ InstancedMesh,+ MeshStandardMaterial,+ Object3D,+ SphereGeometry,+ Vector3,+} from "three";+import { useFrame } from "@react-three/fiber";+import { useServer } from "@/services";+import { Point3d } from "../helpers/viewport3d-types";+import { useViewportState } from "../scene/ViewportStateContext";+import { COLORS } from "../helpers/colors";++const MAX_POINTS = 1024;+const DUMMY = new Object3D();+const FAR_AWAY = new Vector3(1e5, 1e5, 1e5);++const RAW_RADIUS = 0.08;+const FILTERED_RADIUS = 0.12;++interface KeypointLayerProps {+ subscribeKey: "subscribeToKeypointsRaw" | "subscribeToKeypointsFiltered";+ color: Color;+ radius: number;+ statsKey: "keypointsRaw" | "keypointsFiltered";+}++function KeypointLayer({ subscribeKey, color, radius, statsKey }: KeypointLayerProps) {+ const server = useServer();+ const { statsRef } = useViewportState();+ const meshRef = useRef<InstancedMesh>(null);+ const pointsRef = useRef<Record<string, Point3d>>({}); // plain object, matches wire format+ const dirtyRef = useRef(false);+ const nameToIdx = useRef<Map<string, number>>(new Map());+ const nextIdx = useRef(0);++ const geo = useMemo(() => new SphereGeometry(100, 8, 6), []);+ const mat = useMemo(() => new MeshStandardMaterial({+ color: "#ffffff",+ roughness: 0.4,+ metalness: 0.3,+ }), []);++ useEffect(() => {+ const subscribeFn = server[subscribeKey];+ return subscribeFn((pts: Record<string, Point3d>) => {+ pointsRef.current = pts;+ dirtyRef.current = true;+ for (const name of Object.keys(pts)) { // was: pts.keys()+ if (!nameToIdx.current.has(name) && nextIdx.current < MAX_POINTS) {+ nameToIdx.current.set(name, nextIdx.current++);+ }+ }+ });+ }, [server, subscribeKey]);++ useEffect(() => {+ const mesh = meshRef.current;+ if (!mesh) return;+ for (let i = 0; i < MAX_POINTS; i++) {+ DUMMY.position.copy(FAR_AWAY);+ DUMMY.scale.set(0, 0, 0);+ DUMMY.updateMatrix();+ mesh.setMatrixAt(i, DUMMY.matrix);+ mesh.setColorAt(i, COLORS.hidden);+ }+ mesh.instanceMatrix.needsUpdate = true;+ if (mesh.instanceColor) mesh.instanceColor.needsUpdate = true;+ mesh.count = MAX_POINTS;+ }, []);++ useFrame(() => {+ const mesh = meshRef.current;+ if (!mesh || !dirtyRef.current) return;+ const pts = pointsRef.current;+ let count = 0;++ for (const [name, idx] of nameToIdx.current) {+ const pt = pts[name]; // was: pts.get(name)+ if (pt) {+ DUMMY.position.set(pt.x, pt.y, pt.z);+ DUMMY.scale.setScalar(radius);+ mesh.setColorAt(idx, color);+ count++;+ } else {+ // Not in this packet β hide it+ DUMMY.position.copy(FAR_AWAY);+ DUMMY.scale.set(0, 0, 0);+ mesh.setColorAt(idx, COLORS.hidden);+ }+ DUMMY.updateMatrix();+ mesh.setMatrixAt(idx, DUMMY.matrix);+ }++ mesh.instanceMatrix.needsUpdate = true;+ if (mesh.instanceColor) mesh.instanceColor.needsUpdate = true;+ dirtyRef.current = false;+ statsRef.current[statsKey] = count;+ });++ return (+ <instancedMesh ref={meshRef} args={[geo, mat, MAX_POINTS]} frustumCulled={false} />+ );+}++export function KeypointsRenderer() {+ const { visibility } = useViewportState();++ return (+ <>+ {visibility.keypointsRaw && (+ <KeypointLayer+ subscribeKey="subscribeToKeypointsRaw"+ color={COLORS.raw}+ radius={RAW_RADIUS}+ statsKey="keypointsRaw"+ />+ )}+ {visibility.keypointsFiltered && (+ <KeypointLayer+ subscribeKey="subscribeToKeypointsFiltered"+ color={COLORS.filtered}+ radius={FILTERED_RADIUS}+ statsKey="keypointsFiltered"+ />+ )}+ </>+ );+}diff --git a/freemocap-ui/src/components/viewport3d/renderers/RigidBodyRenderer.tsx b/freemocap-ui/src/components/viewport3d/renderers/RigidBodyRenderer.tsxnew file mode 100644index 00000000..b1c7a0e4--- /dev/null+++ b/freemocap-ui/src/components/viewport3d/renderers/RigidBodyRenderer.tsx@@ -0,0 +1,164 @@+import { useEffect, useMemo, useRef } from "react";+import {+ CylinderGeometry,+ Group,+ InstancedMesh,+ MeshStandardMaterial,+ Object3D,+ Vector3,+} from "three";+import { useFrame } from "@react-three/fiber";+import { useServer } from "@/services";+import { RigidBodyPose } from "../helpers/viewport3d-types";+import { useViewportState } from "../scene/ViewportStateContext";+import { COLORS } from "../helpers/colors";++const MAX_BODIES = 128;+const DUMMY = new Object3D();+const FAR_AWAY = new Vector3(1e5, 1e5, 1e5);++/** Shaft radius for the main Z-axis cylinder. */+const SHAFT_RADIUS = 0.012;+/** Crossbar radius and length for X/Y axis indicators. */+const CROSSBAR_RADIUS = 0.006;+const CROSSBAR_LENGTH = 0.04;++/**+ * Renders rigid body segments as instanced cylinders along local Z,+ * with small crossbars at the origin showing X (red) and Y (green) axes.+ *+ * Uses 3 instanced meshes:+ * - main shafts (Z axis, bone color)+ * - X crossbars (red)+ * - Y crossbars (green)+ */+export function RigidBodyRenderer() {+ const { subscribeToRigidBodies } = useServer();+ const { statsRef } = useViewportState();++ const shaftRef = useRef<InstancedMesh>(null);+ const xBarRef = useRef<InstancedMesh>(null);+ const yBarRef = useRef<InstancedMesh>(null);++ const posesRef = useRef<Map<string, RigidBodyPose>>(new Map());+ const nameToIdx = useRef<Map<string, number>>(new Map());+ const nextIdx = useRef(0);+ const dirtyRef = useRef(false);++ // Shaft geometry: unit cylinder along +Z, origin at bottom+ const shaftGeo = useMemo(() => {+ const g = new CylinderGeometry(1, 1, 1, 6, 1);+ g.rotateX(Math.PI / 2); // Y β Z+ g.translate(0, 0, 0.5); // origin at bottom+ return g;+ }, []);++ // Crossbar geometry: short cylinder along +X (will be rotated for Y)+ const crossbarGeo = useMemo(() => {+ const g = new CylinderGeometry(1, 1, 1, 4, 1);+ // leave along Y β we'll orient per-instance+ return g;+ }, []);++ const shaftMat = useMemo(() => new MeshStandardMaterial({+ color: "#ffffff", roughness: 0.5, metalness: 0.2, transparent: true, opacity: 0.7,+ }), []);+ const xMat = useMemo(() => new MeshStandardMaterial({ color: COLORS.rigidBodyX }), []);+ const yMat = useMemo(() => new MeshStandardMaterial({ color: COLORS.rigidBodyY }), []);++ useEffect(() => {+ return subscribeToRigidBodies((newPoses: Map<string, RigidBodyPose>) => {+ posesRef.current = newPoses;+ dirtyRef.current = true;+ for (const key of newPoses.keys()) {+ if (!nameToIdx.current.has(key) && nextIdx.current < MAX_BODIES) {+ nameToIdx.current.set(key, nextIdx.current++);+ }+ }+ });+ }, [subscribeToRigidBodies]);++ // Init all hidden+ useEffect(() => {+ for (const mesh of [shaftRef.current, xBarRef.current, yBarRef.current]) {+ if (!mesh) continue;+ for (let i = 0; i < MAX_BODIES; i++) {+ DUMMY.position.copy(FAR_AWAY);+ DUMMY.scale.set(0, 0, 0);+ DUMMY.updateMatrix();+ mesh.setMatrixAt(i, DUMMY.matrix);+ if (mesh === shaftRef.current) mesh.setColorAt(i, COLORS.hidden);+ }+ mesh.instanceMatrix.needsUpdate = true;+ if (mesh.instanceColor) mesh.instanceColor.needsUpdate = true;+ mesh.count = MAX_BODIES;+ }+ }, []);++ useFrame(() => {+ const shaft = shaftRef.current;+ const xBar = xBarRef.current;+ const yBar = yBarRef.current;+ if (!shaft || !xBar || !yBar || !dirtyRef.current) return;++ const poses = posesRef.current;+ let count = 0;++ for (const [key, idx] of nameToIdx.current) {+ const pose = poses.get(key);++ if (pose) {+ const [px, py, pz] = pose.position;+ const [qw, qx, qy, qz] = pose.orientation;+ const [sx, sy, sz] = pose.scale;++ // Main shaft along local Z+ DUMMY.position.set(px, py, pz);+ DUMMY.quaternion.set(qx, qy, qz, qw);+ DUMMY.scale.set(SHAFT_RADIUS, SHAFT_RADIUS, sz);+ DUMMY.updateMatrix();+ shaft.setMatrixAt(idx, DUMMY.matrix);+ shaft.setColorAt(idx, COLORS.rigidBody);++ // X crossbar: rotate 90Β° around Z from the bone frame, scale along local Y+ DUMMY.scale.set(CROSSBAR_RADIUS, CROSSBAR_LENGTH, CROSSBAR_RADIUS);+ DUMMY.updateMatrix();+ // We need to apply an additional 90Β° rotation around Z for the X bar+ // Simpler: just set scale to put length along X+ DUMMY.scale.set(CROSSBAR_LENGTH, CROSSBAR_RADIUS, CROSSBAR_RADIUS);+ DUMMY.updateMatrix();+ xBar.setMatrixAt(idx, DUMMY.matrix);++ // Y crossbar+ DUMMY.scale.set(CROSSBAR_RADIUS, CROSSBAR_LENGTH, CROSSBAR_RADIUS);+ DUMMY.updateMatrix();+ yBar.setMatrixAt(idx, DUMMY.matrix);++ count++;+ } else {+ DUMMY.position.copy(FAR_AWAY);+ DUMMY.scale.set(0, 0, 0);+ DUMMY.updateMatrix();+ shaft.setMatrixAt(idx, DUMMY.matrix);+ shaft.setColorAt(idx, COLORS.hidden);+ xBar.setMatrixAt(idx, DUMMY.matrix);+ yBar.setMatrixAt(idx, DUMMY.matrix);+ }+ }++ for (const mesh of [shaft, xBar, yBar]) {+ mesh.instanceMatrix.needsUpdate = true;+ if (mesh.instanceColor) mesh.instanceColor.needsUpdate = true;+ }+ dirtyRef.current = false;+ statsRef.current.rigidBodies = count;+ });++ return (+ <>+ <instancedMesh ref={shaftRef} args={[shaftGeo, shaftMat, MAX_BODIES]} frustumCulled={false} />+ <instancedMesh ref={xBarRef} args={[crossbarGeo, xMat, MAX_BODIES]} frustumCulled={false} />+ <instancedMesh ref={yBarRef} args={[crossbarGeo, yMat, MAX_BODIES]} frustumCulled={false} />+ </>+ );+}diff --git a/freemocap-ui/src/components/viewport3d/scene/SceneCamera.tsx b/freemocap-ui/src/components/viewport3d/scene/SceneCamera.tsxnew file mode 100644index 00000000..b1847b9d--- /dev/null+++ b/freemocap-ui/src/components/viewport3d/scene/SceneCamera.tsx@@ -0,0 +1,31 @@+import { useCallback, useEffect } from "react";+import { CameraControls } from "@react-three/drei";+import type CameraControlsImpl from "camera-controls";+import { Box3, Vector3 } from "three";+import { useServer } from "@/services";+import { Point3d } from "../helpers/viewport3d-types";+import { RefObject } from "react";++const PADDING = 0.5;++interface SceneCameraProps {+ controlsRef: RefObject<CameraControlsImpl>;+}++export function SceneCamera({ controlsRef }: SceneCameraProps) {+ return <CameraControls ref={controlsRef} makeDefault />;+}++/** Fit camera to bounding box of all keypoints. */+export function fitCameraToPoints(+ controls: CameraControlsImpl | null,+ points: Map<string, Point3d>,+): void {+ if (!controls || points.size === 0) return;+ const box = new Box3();+ for (const pt of points.values()) {+ box.expandByPoint(new Vector3(pt.x, pt.y, pt.z));+ }+ box.expandByScalar(PADDING);+ controls.fitToBox(box, true);+}diff --git a/freemocap-ui/src/components/viewport3d/scene/SceneEnvironment.tsx b/freemocap-ui/src/components/viewport3d/scene/SceneEnvironment.tsxnew file mode 100644index 00000000..86f50376--- /dev/null+++ b/freemocap-ui/src/components/viewport3d/scene/SceneEnvironment.tsx@@ -0,0 +1,28 @@+import { Grid } from "@react-three/drei";+import { useViewportState } from "./ViewportStateContext";++export function SceneEnvironment() {+ const { visibility } = useViewportState();+ if (!visibility.environment) return null;++ return (+ <>+ <ambientLight intensity={0.6} />+ <directionalLight position={[3e3, 3e3, 3e3]} intensity={0.5} />+ <directionalLight position={[-3e3, -2e3, -3e3]} intensity={0.3} />+ <Grid+ renderOrder={-1}+ position={[0, -0.01, 0]}+ infiniteGrid+ cellSize={1000}+ cellThickness={0.5}+ sectionSize={50}+ sectionThickness={1}+ // @ts-ignore+ sectionColor={[0.1, 0, 0.1]}+ fadeDistance={10000}+ />+ <axesHelper scale={100} />+ </>+ );+}diff --git a/freemocap-ui/src/components/viewport3d/scene/ViewportOverlay.tsx b/freemocap-ui/src/components/viewport3d/scene/ViewportOverlay.tsxnew file mode 100644index 00000000..b27040f8--- /dev/null+++ b/freemocap-ui/src/components/viewport3d/scene/ViewportOverlay.tsx@@ -0,0 +1,112 @@+import { useCallback, useEffect, useState } from "react";+import {+ Box,+ Checkbox,+ Collapse,+ FormControlLabel,+ IconButton,+ Paper,+ Tooltip,+ Typography,+} from "@mui/material";+import ExpandMoreIcon from "@mui/icons-material/ExpandMore";+import ExpandLessIcon from "@mui/icons-material/ExpandLess";+import CenterFocusStrongIcon from "@mui/icons-material/CenterFocusStrong";+import HomeIcon from "@mui/icons-material/Home";+import { ViewportStats, ViewportVisibility } from "../helpers/viewport3d-types";+import { useViewportState } from "./ViewportStateContext";++interface ViewportOverlayProps {+ onFitCamera: () => void;+ onResetCamera: () => void;+}++export function ViewportOverlay({ onFitCamera, onResetCamera }: ViewportOverlayProps) {+ const { visibility, setVisibility, statsRef } = useViewportState();+ const [stats, setStats] = useState<ViewportStats>({ keypointsRaw: 0, keypointsFiltered: 0, rigidBodies: 0, facePoints: 0 });+ const [expanded, setExpanded] = useState(false);++ // Poll stats from the mutable ref at ~4 Hz (no re-render coupling to the frame loop)+ useEffect(() => {+ const id = setInterval(() => setStats({ ...statsRef.current }), 250);+ return () => clearInterval(id);+ }, [statsRef]);++ const toggle = useCallback((key: keyof ViewportVisibility) => {+ setVisibility(prev => ({ ...prev, [key]: !prev[key] }));+ }, [setVisibility]);++ return (+ <>+ {/* Top-left: visibility + stats */}+ <Paper+ sx={{+ position: "absolute", top: 8, left: 8,+ p: 1, bgcolor: "rgba(0,0,0,0.75)", color: "#ccc",+ minWidth: 180, userSelect: "none",+ }}+ elevation={3}+ >+ <Box sx={{ display: "flex", alignItems: "center", justifyContent: "space-between" }}>+ <Typography variant="caption" fontWeight="bold">Viewport</Typography>+ <IconButton size="small" onClick={() => setExpanded(e => !e)} sx={{ color: "#ccc" }}>+ {expanded ? <ExpandLessIcon fontSize="small" /> : <ExpandMoreIcon fontSize="small" />}+ </IconButton>+ </Box>++ <VisToggle label="Environment" checked={visibility.environment} onChange={() => toggle("environment")} />+ <VisToggle label={`Raw (${stats.keypointsRaw})`} checked={visibility.keypointsRaw} onChange={() => toggle("keypointsRaw")} />+ <VisToggle label={`Filtered (${stats.keypointsFiltered})`} checked={visibility.keypointsFiltered} onChange={() => toggle("keypointsFiltered")} />+ <VisToggle label={`Rigid bodies (${stats.rigidBodies})`} checked={visibility.rigidBodies} onChange={() => toggle("rigidBodies")} />+ <VisToggle label={`Face (${stats.facePoints})`} checked={visibility.face} onChange={() => toggle("face")} />++ <Collapse in={expanded}>+ <Typography variant="caption" sx={{ mt: 1, display: "block", color: "#888" }}>+ Total points: {stats.keypointsRaw + stats.keypointsFiltered + stats.facePoints}+ <br />+ Total bodies: {stats.rigidBodies}+ </Typography>+ </Collapse>+ </Paper>++ {/* Bottom-right: camera buttons */}+ <Box sx={{ position: "absolute", bottom: 8, right: 8, display: "flex", gap: 0.5 }}>+ <Tooltip title="Fit to skeleton (F)">+ <IconButton onClick={onFitCamera} size="small" sx={btnSx}>+ <CenterFocusStrongIcon fontSize="small" />+ </IconButton>+ </Tooltip>+ <Tooltip title="Reset view">+ <IconButton onClick={onResetCamera} size="small" sx={btnSx}>+ <HomeIcon fontSize="small" />+ </IconButton>+ </Tooltip>+ </Box>++ {/* Bottom-left: hint */}+ <Typography+ variant="caption"+ sx={{ position: "absolute", bottom: 8, left: 8, color: "#666", pointerEvents: "none" }}+ >+ Rotate: drag Β· Zoom: scroll Β· Pan: right-drag+ </Typography>+ </>+ );+}++function VisToggle({ label, checked, onChange }: { label: string; checked: boolean; onChange: () => void }) {+ return (+ <FormControlLabel+ sx={{ m: 0, ml: -0.5, "& .MuiTypography-root": { fontSize: "0.7rem" } }}+ control={<Checkbox size="small" checked={checked} onChange={onChange} sx={{ p: 0.3, color: "#888", "&.Mui-checked": { color: "#aaa" } }} />}+ label={label}+ />+ );+}++const btnSx = {+ bgcolor: "rgba(0,0,0,0.6)",+ color: "#ccc",+ boxShadow: 2,+ "&:hover": { bgcolor: "rgba(60,60,60,0.8)" },+};diff --git a/freemocap-ui/src/components/viewport3d/scene/ViewportStateContext.tsx b/freemocap-ui/src/components/viewport3d/scene/ViewportStateContext.tsxnew file mode 100644index 00000000..ae41ab47--- /dev/null+++ b/freemocap-ui/src/components/viewport3d/scene/ViewportStateContext.tsx@@ -0,0 +1,35 @@+import React, { createContext, useCallback, useContext, useMemo, useRef, useState } from "react";+import { DEFAULT_VISIBILITY, ViewportStats, ViewportVisibility } from "../helpers/viewport3d-types";++interface ViewportStateContextValue {+ visibility: ViewportVisibility;+ setVisibility: React.Dispatch<React.SetStateAction<ViewportVisibility>>;+ /** Mutable stats ref β renderers write here each frame, overlay reads on a poll timer. */+ statsRef: React.MutableRefObject<ViewportStats>;+}++const ViewportStateContext = createContext<ViewportStateContextValue | null>(null);++export function ViewportStateProvider({ children }: { children: React.ReactNode }) {+ const [visibility, setVisibility] = useState<ViewportVisibility>(DEFAULT_VISIBILITY);+ const statsRef = useRef<ViewportStats>({+ keypointsRaw: 0,+ keypointsFiltered: 0,+ rigidBodies: 0,+ facePoints: 0,+ });++ const value = useMemo(() => ({ visibility, setVisibility, statsRef }), [visibility]);++ return (+ <ViewportStateContext.Provider value={value}>+ {children}+ </ViewportStateContext.Provider>+ );+}++export function useViewportState() {+ const ctx = useContext(ViewportStateContext);+ if (!ctx) throw new Error("useViewportState must be used within ViewportStateProvider");+ return ctx;+}diff --git a/freemocap-ui/src/services/server/ServerContextProvider.tsx b/freemocap-ui/src/services/server/ServerContextProvider.tsxindex 1693c4d6..66ca14f6 100644--- a/freemocap-ui/src/services/server/ServerContextProvider.tsx+++ b/freemocap-ui/src/services/server/ServerContextProvider.tsx@@ -5,9 +5,8 @@ import {ConnectionState, WebSocketConnection} from "@/services/server/server-helimport {FrameProcessor} from "@/services/server/server-helpers/frame-processor/frame-processor";import {CanvasManager} from "@/services/server/server-helpers/canvas-manager";import {serverUrls} from "@/services";-import {DetailedFramerate, FramerateStore} from "@/services/server/server-helpers/framerate-store";-import {LogRecord, LogStore} from "@/services/server/server-helpers/log-store";-import {Point3d, RigidBodyPose} from "@/components/viewport3d/viewport3d-types";+import {FramerateStore} from "@/services/server/server-helpers/framerate-store";+import {LogStore} from "@/services/server/server-helpers/log-store";import {OverlayManager} from "@/services/server/server-helpers/image-overlay/overlay-renderer-factory";import {CharucoObservation} from "@/services/server/server-helpers/image-overlay/charuco-types";import {MediapipeObservation} from "@/services/server/server-helpers/image-overlay/mediapipe-types";@@ -16,6 +15,7 @@ import {isFrontendPayload,isLogRecord} from "@/services/server/server-helpers/websocket-message-types";+import {Point3d, RigidBodyPose} from "@/components/viewport3d";interface ServerContextValue {isConnected: boolean;@@ -29,9 +29,10 @@ interface ServerContextValue {getLogStore: () => LogStore;connectedCameraIds: string[];updateServerConnection: (host: string, port: number) => void;- subscribeToTrackedPoints: (cb: (points: Map<string, Point3d>) => void) => () => void;+ subscribeToKeypointsRaw: (cb: (points: Record<string, Point3d>) => void) => () => void;+ subscribeToKeypointsFiltered: (cb: (points: Record<string, Point3d>) => void) => () => void;subscribeToRigidBodies: (cb: (poses: Map<string, RigidBodyPose>) => void) => () => void;- getLatestTrackedPoints: () => Map<string, Point3d>;+ getLatestKeypointsRaw: () => Record<string, Point3d>;}const ServerContext = createContext<ServerContextValue | null>(null);@@ -66,10 +67,13 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({childrconst serverFpsRef = useRef<number | null>(null);// 3D data refs and subscriber sets- const trackedPointsRef = useRef<Map<string, Point3d>>(new Map());+ // Plain Record β matches JSON.parse output directly, no remapping needed.+ const trackedPointsRef = useRef<Record<string, Point3d>>({});const rigidBodiesRef = useRef<Map<string, RigidBodyPose>>(new Map());- const trackedPointsSubscribersRef = useRef<Set<(points: Map<string, Point3d>) => void>>(new Set());+ const trackedPointsSubscribersRef = useRef<Set<(points: Record<string, Point3d>) => void>>(new Set());const rigidBodiesSubscribersRef = useRef<Set<(poses: Map<string, RigidBodyPose>) => void>>(new Set());+ const keypointsFilteredRef = useRef<Record<string, Point3d>>({});+ const keypointsFilteredSubscribersRef = useRef<Set<(points: Record<string, Point3d>) => void>>(new Set());// Holds the latest binary payload received from the WebSocket.@@ -131,8 +135,9 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({childrpendingPayloadRef.current = null;lastCameraIdsRef.current = [];framerateStoreRef.current.clear();- trackedPointsRef.current = new Map();+ trackedPointsRef.current = {};rigidBodiesRef.current = new Map();+ keypointsFilteredRef.current = {};latestCharucoRef.current.clear();latestMediapipeRef.current.clear();overlayManagerRef.current?.clearAll();@@ -237,14 +242,14 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({childrif (event.data instanceof ArrayBuffer) {pendingPayloadRef.current = event.data;}- // Handle text/JSON messages (logs, framerate updates, frontend payloads, etc.)+ // Handle text/JSON messages (logs, framerate updates, frontend payloads, etc.)else if (typeof event.data === 'string') {// Skip heartbeat pong responses β they're plain text, not JSONif (event.data === 'pong') return;try {const jsonData = JSON.parse(event.data);- console.log(JSON.stringify(jsonData, null, 2));+// Handle log recordsif (isLogRecord(jsonData)) {logStoreRef.current.add(jsonData);@@ -270,45 +275,30 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({childr}}+ // Keypoints arrive as plain JSON objects β assign directly, no remapping.if (jsonData.keypoints_raw) {- const pointsMap = new Map<string, Point3d>();- for (const [name, pt] of Object.entries(jsonData.keypoints_raw)) {- const p = pt as { x: number; y: number; z: number };- pointsMap.set(name, {x: p.x, y: p.y, z: p.z});- }- trackedPointsRef.current = pointsMap;+ trackedPointsRef.current = jsonData.keypoints_raw as Record<string, Point3d>;for (const cb of trackedPointsSubscribersRef.current) {- cb(pointsMap);+ cb(trackedPointsRef.current);}}- }+ if (jsonData.keypoints_filtered) {+ keypointsFilteredRef.current = jsonData.keypoints_filtered as Record<string, Point3d>;+ for (const cb of keypointsFilteredSubscribersRef.current) {+ cb(keypointsFilteredRef.current);+ }+ }- // // Handle tracked 3D points- // else if (jsonData.tracked_points3d && typeof jsonData.tracked_points3d === 'object') {- // const pointsMap = new Map<string, Point3d>();- // for (const [name, pt] of Object.entries(jsonData.tracked_points3d)) {- // const p = pt as { x: number; y: number; z: number };- // pointsMap.set(name, {x: p.x, y: p.y, z: p.z});- // }- // trackedPointsRef.current = pointsMap;- // for (const cb of trackedPointsSubscribersRef.current) {- // cb(pointsMap);- // }- // }- // // Handle rigid body poses- // else if (jsonData.rigid_body_poses && typeof jsonData.rigid_body_poses === 'object') {- // const posesMap = new Map<string, RigidBodyPose>();- // for (const [key, pose] of Object.entries(jsonData.rigid_body_poses)) {- // posesMap.set(key, pose as RigidBodyPose);- // }- // rigidBodiesRef.current = posesMap;- // for (const cb of rigidBodiesSubscribersRef.current) {- // cb(posesMap);- // }- // }- // Handle other message types (silently ignored to avoid- // retaining object references in the DevTools console)+ if (jsonData.rigid_body_poses) {+ const posesMap = new Map<string, RigidBodyPose>();+ for (const [key, pose] of Object.entries(jsonData.rigid_body_poses)) {+ posesMap.set(key, pose as RigidBodyPose);+ }+ rigidBodiesRef.current = posesMap;+ for (const cb of rigidBodiesSubscribersRef.current) cb(posesMap);+ }+ }} catch (error) {console.error('Error parsing JSON message:', error);}@@ -364,11 +354,14 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({childrreturn logStoreRef.current;}, []);- const subscribeToTrackedPoints = useCallback((cb: (points: Map<string, Point3d>) => void): () => void => {+ const subscribeToKeypointsRaw = useCallback((cb: (points: Record<string, Point3d>) => void): () => void => {trackedPointsSubscribersRef.current.add(cb);- return () => {- trackedPointsSubscribersRef.current.delete(cb);- };+ return () => { trackedPointsSubscribersRef.current.delete(cb); };+ }, []);++ const subscribeToKeypointsFiltered = useCallback((cb: (points: Record<string, Point3d>) => void): () => void => {+ keypointsFilteredSubscribersRef.current.add(cb);+ return () => { keypointsFilteredSubscribersRef.current.delete(cb); };}, []);const subscribeToRigidBodies = useCallback((cb: (poses: Map<string, RigidBodyPose>) => void): () => void => {@@ -378,7 +371,7 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({childr};}, []);- const getLatestTrackedPoints = useCallback((): Map<string, Point3d> => {+ const getLatestKeypointsRaw = useCallback((): Record<string, Point3d> => {return trackedPointsRef.current;}, []);@@ -408,10 +401,11 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({childrgetLogStore,connectedCameraIds,updateServerConnection,- subscribeToTrackedPoints,+ subscribeToKeypointsRaw,+ subscribeToKeypointsFiltered,subscribeToRigidBodies,- getLatestTrackedPoints,- }), [isConnected, connectedCameraIds, connect, disconnect, sendWebsocketMessage, setCanvasForCamera, getFps, getServerFps, getFramerateStore, getLogStore, updateServerConnection, subscribeToTrackedPoints, subscribeToRigidBodies, getLatestTrackedPoints]);+ getLatestKeypointsRaw,+ }), [isConnected, connectedCameraIds, connect, disconnect, sendWebsocketMessage, setCanvasForCamera, getFps, getServerFps, getFramerateStore, getLogStore, updateServerConnection, subscribeToKeypointsRaw, subscribeToKeypointsFiltered, subscribeToRigidBodies, getLatestKeypointsRaw]);return (<ServerContext.Provider value={contextValue}>diff --git a/freemocap-ui/src/services/server/server-helpers/websocket-message-types.ts b/freemocap-ui/src/services/server/server-helpers/websocket-message-types.tsindex 5afca68f..4505566c 100644--- a/freemocap-ui/src/services/server/server-helpers/websocket-message-types.ts+++ b/freemocap-ui/src/services/server/server-helpers/websocket-message-types.ts@@ -12,7 +12,7 @@ import {ModelInfo} from "@/services/server/server-helpers/image-overlay/image-ovimport {OverlayRendererFactory} from "@/services/server/server-helpers/image-overlay/overlay-renderer-factory";import {DetailedFramerate} from "@/services/server/server-helpers/framerate-store";import {LogRecord} from "@/services/server/server-helpers/log-store";-import {Point3d, RigidBodyPose} from "@/components/viewport3d/viewport3d-types";+import {Point3d, RigidBodyPose} from "@/components/viewport3d";// Type guard to check if a message is a log recordexport function isLogRecord(data: any): data is LogRecord {diff --git a/freemocap/api/websocket/websocket_server.py b/freemocap/api/websocket/websocket_server.pyindex 403479e3..6e924560 100644--- a/freemocap/api/websocket/websocket_server.py+++ b/freemocap/api/websocket/websocket_server.py@@ -128,65 +128,39 @@ class WebsocketServer:return self.last_received_frontend_confirmation >= self.last_sent_frame_numberasync def _frontend_image_relay(self) -> None:- """Relay image payloads from shared memory to the frontend via WebSocket."""logger.info("Starting frontend image payload relay...")try:skipped_previous = Falsewhile self.should_continue:await await_10ms()- if self.check_frame_acknowledgment_status():- if skipped_previous:- skipped_previous = False- else:- try:- frontend_payloads = self._app.get_latest_frontend_payloads(- if_newer_than=self.last_sent_frame_number- )- except IndexError:- logger.warning(- f"Ring buffer overwrite detected (last_sent_frame_number={self.last_sent_frame_number}). "- f"Resetting to latest frame."- )- self.last_sent_frame_number = -1- continue+ if not self.check_frame_acknowledgment_status():+ skipped_previous = True+ backpressure = self.last_sent_frame_number - self.last_received_frontend_confirmation+ if backpressure > BACKPRESSURE_WARNING_THRESHOLD and backpressure % BACKPRESSURE_WARNING_THRESHOLD == 0:+ logger.trace(f"Backpressure: {backpressure} frames unacknowledged")+ continue- for pipeline_id, (image_payload_bytes, frontend_payload) in frontend_payloads.items():- frame_number = None- if not image_payload_bytes and not frontend_payload:- continue- if frontend_payload:- if not isinstance(frontend_payload, FrontendPayload):- frame_number = frontend_payload- frontend_payload = None- else:- frame_number = frontend_payload.frame_number- async with self._send_lock:- await self.websocket.send_json(frontend_payload.model_dump())+ if skipped_previous:+ skipped_previous = False+ continue- if image_payload_bytes:- if not isinstance(image_payload_bytes, (bytes, bytearray)):- raise TypeError(- f"Invalid payload bytes on frame {frame_number} - "- f"got type {type(image_payload_bytes).__name__}"- )- async with self._send_lock:- await self.websocket.send_bytes(image_payload_bytes)+ try:+ packets = self._app.get_latest_frontend_payloads(if_newer_than=self.last_sent_frame_number)+ except IndexError:+ logger.warning("Ring buffer overwrite β resetting to latest frame")+ self.last_sent_frame_number = -1+ continue+ for packet in packets.values():+ if packet.frontend_payload is not None:+ async with self._send_lock:+ await self.websocket.send_json(packet.frontend_payload.model_dump())- if frame_number is not None:- self.last_sent_frame_number = frame_number- else:- skipped_previous = True- backpressure = self.last_sent_frame_number - self.last_received_frontend_confirmation- if (- backpressure > BACKPRESSURE_WARNING_THRESHOLD- and backpressure % BACKPRESSURE_WARNING_THRESHOLD == 0- ):- logger.trace(- f"Backpressure detected: {backpressure} frames not acknowledged by frontend! "- f"Last sent frame: {self.last_sent_frame_number}, "- f"last received confirmation: {self.last_received_frontend_confirmation}"- )+ if packet.image_bytes is not None:+ async with self._send_lock:+ await self.websocket.send_bytes(packet.image_bytes)++ self.last_sent_frame_number = packet.frame_numberexcept WebSocketDisconnect:logger.api("Client disconnected, ending Frontend Image relay task...")diff --git a/freemocap/app/freemocap_application.py b/freemocap/app/freemocap_application.pyindex 4091f731..10da7cae 100644--- a/freemocap/app/freemocap_application.py+++ b/freemocap/app/freemocap_application.py@@ -21,15 +21,13 @@ from freemocap.core.pipeline.realtime.realtime_pipeline_manager import RealtimePfrom freemocap.core.tasks.calibration.calibration_task_config import PosthocCalibrationPipelineConfigfrom freemocap.core.tasks.mocap.mocap_task_config import PosthocMocapPipelineConfigfrom freemocap.core.types.type_overloads import PipelineIdString, FrameNumberInt-from freemocap.core.viz.frontend_payload import FrontendPayload+from freemocap.core.viz.frontend_payload import FrontendPayload, FrontendImagePacketlogger = logging.getLogger(__name__)-@dataclassclass FreemocapApplication:-global_kill_flag: multiprocessing.Valueworker_registry: WorkerRegistryrealtime_pipeline_manager: RealtimePipelineManager@@ -65,8 +63,8 @@ class FreemocapApplication:# ------------------------------------------------------------------async def create_or_update_realtime_pipeline(- self,- pipeline_config: RealtimePipelineConfig,+ self,+ pipeline_config: RealtimePipelineConfig,) -> RealtimePipeline:existing = self.realtime_pipeline_manager.get_pipeline_by_camera_ids(camera_ids=pipeline_config.camera_ids,@@ -91,9 +89,9 @@ class FreemocapApplication:# ------------------------------------------------------------------async def create_posthoc_calibration_pipeline(- self,- recording_info: RecordingInfo,- calibration_config: PosthocCalibrationPipelineConfig,+ self,+ recording_info: RecordingInfo,+ calibration_config: PosthocCalibrationPipelineConfig,) -> PosthocPipeline:pipeline = self.posthoc_pipeline_manager.create_calibration_pipeline(recording_info=recording_info,@@ -103,9 +101,9 @@ class FreemocapApplication:return pipelineasync def create_posthoc_mocap_pipeline(- self,- recording_info: RecordingInfo,- mocap_config: PosthocMocapPipelineConfig,+ self,+ recording_info: RecordingInfo,+ mocap_config: PosthocMocapPipelineConfig,) -> PosthocPipeline:pipeline = self.posthoc_pipeline_manager.create_mocap_pipeline(recording_info=recording_info,@@ -141,26 +139,31 @@ class FreemocapApplication:# ------------------------------------------------------------------def get_latest_frontend_payloads(- self,- if_newer_than: FrameNumberInt,- ) -> dict[str, tuple[bytes, int]] | dict[str, tuple[bytes | None, FrontendPayload | None]]:- # Clean up completed posthoc pipelines (releases relay threads + queues)+ self,+ if_newer_than: FrameNumberInt,+ ) -> dict[str, FrontendImagePacket]:self.posthoc_pipeline_manager.evict_completed()realtime_pipelines = self.realtime_pipeline_manager.pipelines- if len(realtime_pipelines) == 0 or all(- not p.alive for p in realtime_pipelines.values()- ):- cg_payloads = self.camera_group_manager.get_latest_frontend_payloads(- if_newer_than=if_newer_than,- )- return {- camera_group_id: (payload[-1], payload[0])- for camera_group_id, payload in cg_payloads.items()- }-+ active_pipelines = [p for p in realtime_pipelines.values() if p.alive]++ if not active_pipelines:+ # Camera-only path+ result = {}+ for cg_id, payload in self.camera_group_manager.get_latest_frontend_payloads(+ if_newer_than=if_newer_than+ ).items():+ frame_number, _timestamp, image_bytes = payload # unpack the known tuple shape+ result[cg_id] = FrontendImagePacket(+ image_bytes=image_bytes,+ frame_number=frame_number,+ frontend_payload=None,+ )+ return result++ # Pipeline path β delegate to manager, which also returns FrontendImagePacketreturn self.realtime_pipeline_manager.get_latest_frontend_payloads(- if_newer_than=if_newer_than,+ if_newer_than=if_newer_than)# ------------------------------------------------------------------@@ -182,7 +185,6 @@ class FreemocapApplication:self.posthoc_pipeline_manager.shutdown()-FREEMOCAP_APP: FreemocapApplication | None = Nonediff --git a/freemocap/core/pipeline/realtime/realtime_pipeline.py b/freemocap/core/pipeline/realtime/realtime_pipeline.pyindex c83c88b3..3afb353f 100644--- a/freemocap/core/pipeline/realtime/realtime_pipeline.py+++ b/freemocap/core/pipeline/realtime/realtime_pipeline.py@@ -24,7 +24,7 @@ from freemocap.core.pipeline.realtime.camera_node import CameraNodefrom freemocap.core.pipeline.realtime.realtime_aggregator_node import RealtimeAggregatorNodefrom freemocap.core.pipeline.realtime.realtime_pipeline_config import RealtimePipelineConfigfrom freemocap.core.types.type_overloads import PipelineIdString, TopicSubscriptionQueue, FrameNumberInt-from freemocap.core.viz.frontend_payload import FrontendPayload+from freemocap.core.viz.frontend_payload import FrontendPayload, FrontendImagePacketfrom freemocap.pubsub.pubsub_manager import PubSubTopicManagerfrom freemocap.pubsub.pubsub_topics import (AggregationNodeOutputTopic,@@ -62,8 +62,8 @@ class RealtimePipeline:if not self.started:return Falsereturn (- all(node.is_alive for node in self.camera_nodes.values())- and self.aggregation_node.is_alive+ all(node.is_alive for node in self.camera_nodes.values())+ and self.aggregation_node.is_alive)@property@@ -80,11 +80,11 @@ class RealtimePipeline:@classmethoddef create(- cls,- *,- camera_group: CameraGroup,- worker_registry: WorkerRegistry,- pipeline_config: RealtimePipelineConfig,+ cls,+ *,+ camera_group: CameraGroup,+ worker_registry: WorkerRegistry,+ pipeline_config: RealtimePipelineConfig,) -> "RealtimePipeline":global_kill_flag = camera_group.ipc.global_kill_flag@@ -185,21 +185,17 @@ class RealtimePipeline:requested_configs=camera_configs,)- def get_latest_frontend_payload(- self,- if_newer_than: FrameNumberInt,- ) -> None | tuple[Any, None] | tuple[bytes | None, FrontendPayload]:- """- Drain the aggregation output queue and return the latest payload.- Returns None if no new data is available.- """+ def get_latest_frontend_payload(self, if_newer_than: FrameNumberInt, ) -> FrontendImagePacket | None:if not self.alive:if self.camera_group.alive:- _, _, frames_bytearray = self.camera_group.get_latest_frontend_payload(- if_newer_than=if_newer_than,- )- if frames_bytearray is not None:- return (frames_bytearray, None)+ result = self.camera_group.get_latest_frontend_payload(if_newer_than=if_newer_than)+ if result is not None:+ frame_number, _ts, frames_bytes = result+ return FrontendImagePacket(+ image_bytes=frames_bytes,+ frame_number=frame_number,+ frontend_payload=None,+ )return Noneaggregation_output: AggregationNodeOutputMessage | None = None@@ -209,11 +205,26 @@ class RealtimePipeline:if aggregation_output is None:return None- frames_bytearray = self.camera_group.get_frontend_payload_by_frame_number(+ frames_bytes = self.camera_group.get_frontend_payload_by_frame_number(+ frame_number=aggregation_output.frame_number,+ )+ return FrontendImagePacket(+ image_bytes=frames_bytes,+ frame_number=aggregation_output.frame_number,+ frontend_payload=FrontendPayload.from_aggregation_output(aggregation_output),+ )+ aggregation_output: AggregationNodeOutputMessage | None = None+ while not self.aggregation_output_subscription.empty():+ aggregation_output = self.aggregation_output_subscription.get()++ if aggregation_output is None:+ return None, None++ frames_bytes = self.camera_group.get_frontend_payload_by_frame_number(frame_number=aggregation_output.frame_number,)return (- frames_bytearray,+ frames_bytes,FrontendPayload.from_aggregation_output(aggregation_output=aggregation_output),)diff --git a/freemocap/core/pipeline/realtime/realtime_pipeline_manager.py b/freemocap/core/pipeline/realtime/realtime_pipeline_manager.pyindex 2f214c94..faac6a8f 100644--- a/freemocap/core/pipeline/realtime/realtime_pipeline_manager.py+++ b/freemocap/core/pipeline/realtime/realtime_pipeline_manager.py@@ -23,7 +23,7 @@ from freemocap.core.pipeline.abcs.pipeline_manager_abc import PipelineManagerABCfrom freemocap.core.pipeline.realtime.realtime_aggregator_node import RealtimePipelineConfigfrom freemocap.core.pipeline.realtime.realtime_pipeline import RealtimePipelinefrom freemocap.core.types.type_overloads import PipelineIdString, FrameNumberInt-from freemocap.core.viz.frontend_payload import FrontendPayload+from freemocap.core.viz.frontend_payload import FrontendPayload, FrontendImagePacketlogger = logging.getLogger(__name__)@@ -104,15 +104,15 @@ class RealtimePipelineManager(PipelineManagerABC):# ------------------------------------------------------------------def get_latest_frontend_payloads(- self,- if_newer_than: FrameNumberInt,- ) -> dict[PipelineIdString, tuple[bytes | None, FrontendPayload | None]]:- latest: dict[PipelineIdString, tuple[bytes | None, FrontendPayload | None]] = {}+ self,+ if_newer_than: FrameNumberInt,+ ) -> dict[PipelineIdString, FrontendImagePacket]:+ latest: dict[PipelineIdString, FrontendImagePacket] = {}with self.lock:for pipeline_id, pipeline in self.pipelines.items():- output = pipeline.get_latest_frontend_payload(if_newer_than=if_newer_than)- if output is not None:- latest[pipeline_id] = output+ packet = pipeline.get_latest_frontend_payload(if_newer_than=if_newer_than)+ if packet is not None:+ latest[pipeline_id] = packetreturn latest# ------------------------------------------------------------------diff --git a/freemocap/core/viz/frontend_payload.py b/freemocap/core/viz/frontend_payload.pyindex 6cfbbdce..966ec304 100644--- a/freemocap/core/viz/frontend_payload.py+++ b/freemocap/core/viz/frontend_payload.py@@ -12,6 +12,11 @@ from freemocap.pubsub.pubsub_topics import AggregationNodeOutputMessagelogger = logging.getLogger(__name__)+# freemocap/core/viz/frontend_image_packet.py+from dataclasses import dataclass+++class FrontendPayload(BaseModel):"""Complete payload for frontend visualization.@@ -19,17 +24,14 @@ class FrontendPayload(BaseModel):Carries both charuco and mediapipe overlay data separately so bothcan be active simultaneously per camera."""- model_config = ConfigDict(- arbitrary_types_allowed=True,- validate_assignment=False- )- message_type: str = "frontend_payload"+frame_number: intcharuco_overlays: dict[CameraIdString, CharucoOverlayData]skeleton_overlays: dict[CameraIdString, MediapipeOverlayData]keypoints_raw: dict[TrackedPointNameString, Point3d]keypoints_filtered: dict[TrackedPointNameString, Point3d]rigid_body_poses: dict[str, RigidBodyPose]+ message_type: str = "frontend_payload"@classmethoddef from_aggregation_output(@@ -45,3 +47,16 @@ class FrontendPayload(BaseModel):keypoints_filtered=aggregation_output.keypoints_filtered,rigid_body_poses=aggregation_output.rigid_body_poses,)++@dataclass(slots=True, frozen=True)+class FrontendImagePacket:+ """+ Unified output type for all frontend image relay paths.++ `frontend_payload` is None in camera-only mode (no active pipeline).+ `frame_number` is always populated so the relay can track acknowledgment+ without any isinstance checks.+ """+ image_bytes: bytes | None+ frame_number: int+ frontend_payload: FrontendPayload | None # None = camera-only mode