Free Motion Capture for Everyone 💀✨
TypeScript
refactor playback
jonmatthisa125861ea98e
diff --git a/freemocap-ui/package-lock.json b/freemocap-ui/package-lock.jsonindex eb114be3..94b37904 100644--- a/freemocap-ui/package-lock.json+++ b/freemocap-ui/package-lock.json@@ -28,6 +28,7 @@"country-flag-icons": "^1.6.14","d3": "^7.9.0","electron-updater": "^6.8.1",+ "hyparquet": "^1.25.6","i18next-browser-languagedetector": "^8.2.1","js-yaml": "^4.1.1","react-grid-layout": "^2.2.2",@@ -6534,6 +6535,12 @@"node": ">= 14"}},+ "node_modules/hyparquet": {+ "version": "1.25.6",+ "resolved": "https://registry.npmjs.org/hyparquet/-/hyparquet-1.25.6.tgz",+ "integrity": "sha512-Q9W5IjkVch3ZMnYd4qFv2q8suu5Jc36yt7J+zUNM9grwnP1S189icp0jdEQKM5HJvQkTVy8NMiQ8n/dM5QAt1A==",+ "license": "MIT"+ },"node_modules/i18next": {"version": "26.0.3","resolved": "https://registry.npmjs.org/i18next/-/i18next-26.0.3.tgz",diff --git a/freemocap-ui/package.json b/freemocap-ui/package.jsonindex a1685ed2..9dcf4167 100644--- a/freemocap-ui/package.json+++ b/freemocap-ui/package.json@@ -39,6 +39,7 @@"country-flag-icons": "^1.6.14","d3": "^7.9.0","electron-updater": "^6.8.1",+ "hyparquet": "^1.25.6","i18next-browser-languagedetector": "^8.2.1","js-yaml": "^4.1.1","react-grid-layout": "^2.2.2",diff --git a/freemocap-ui/src/components/viewport3d/FileKeypointsSourceProvider.tsx b/freemocap-ui/src/components/viewport3d/FileKeypointsSourceProvider.tsxnew file mode 100644index 00000000..b3ca443f--- /dev/null+++ b/freemocap-ui/src/components/viewport3d/FileKeypointsSourceProvider.tsx@@ -0,0 +1,262 @@+import React, {useEffect, useMemo, useRef} from "react";+import {parquetReadObjects} from "hyparquet";+import {Point3d} from "./helpers/viewport3d-types";+import {+ KeypointsCallback,+ KeypointsSource,+ KeypointsSourceProvider,+} from "./KeypointsSourceContext";+import {serverUrls} from "@/constants/server-urls";++/**+ * KeypointsSource implementation that reads the recording's long-format+ * `freemocap_data_by_frame.parquet` directly in the browser (via hyparquet),+ * pivots long→wide into per-trajectory Float32Array buffers, and emits+ * per-frame point records in lockstep with the playback slider.+ *+ * Design notes:+ * - Parquet is the canonical freemocap data format; decoding it client-side+ * keeps the backend simple (one passthrough file endpoint) and avoids+ * sibling cache artifacts that can drift from the source.+ * - After decode, each trajectory is packed as Float32Array of length+ * frameCount * K * 3, giving O(1) per-frame indexing.+ * - Subscribers receive a pre-allocated scratch record whose Point3d slots+ * are mutated in place each tick (zero GC pressure).+ * - Driven by `currentFrameRef` from usePlaybackController, polled via our+ * own rAF loop. React re-renders are not involved in the hot path.+ */++type TrajectoryAlias = "raw" | "filtered";++const TRAJECTORY_PARQUET_VALUE: Record<TrajectoryAlias, string> = {+ raw: "3d_xyz",+ filtered: "rigid_3d_xyz",+};++interface TrajectoryData {+ names: string[];+ buffer: Float32Array; // length = frameCount * K * 3+ frameCount: number;+ keypointCount: number;+ scratch: Record<string, Point3d>;+ subscribers: Set<KeypointsCallback>;+ lastEmittedFrame: number; // -1 means "never emitted"+}++interface ParquetRow {+ frame: number;+ keypoint: string;+ x: number;+ y: number;+ z: number;+ trajectory: string;+}++function emptyTrajectory(): TrajectoryData {+ return {+ names: [],+ buffer: new Float32Array(0),+ frameCount: 0,+ keypointCount: 0,+ scratch: {},+ subscribers: new Set(),+ lastEmittedFrame: -1,+ };+}++function buildTrajectory(+ rows: ParquetRow[],+ parquetValue: string,+ frameCount: number,+): TrajectoryData {+ // First pass: collect unique keypoint names for this trajectory.+ const nameSet = new Set<string>();+ for (let i = 0; i < rows.length; i++) {+ const row = rows[i];+ if (row.trajectory === parquetValue) nameSet.add(String(row.keypoint));+ }+ if (nameSet.size === 0) return emptyTrajectory();++ const names = Array.from(nameSet).sort();+ const nameToIdx = new Map<string, number>();+ for (let i = 0; i < names.length; i++) nameToIdx.set(names[i], i);++ const K = names.length;+ const buffer = new Float32Array(frameCount * K * 3);+ buffer.fill(NaN);++ // Second pass: scatter rows into the packed buffer.+ for (let i = 0; i < rows.length; i++) {+ const row = rows[i];+ if (row.trajectory !== parquetValue) continue;+ const f = row.frame;+ if (f < 0 || f >= frameCount) continue;+ const k = nameToIdx.get(String(row.keypoint));+ if (k === undefined) continue;+ const off = (f * K + k) * 3;+ buffer[off] = row.x;+ buffer[off + 1] = row.y;+ buffer[off + 2] = row.z;+ }++ const scratch: Record<string, Point3d> = {};+ for (const name of names) scratch[name] = {x: NaN, y: NaN, z: NaN};++ return {+ names,+ buffer,+ frameCount,+ keypointCount: K,+ scratch,+ subscribers: new Set(),+ lastEmittedFrame: -1,+ };+}++function updateScratch(traj: TrajectoryData, frame: number): void {+ const K = traj.keypointCount;+ if (K === 0) return;+ const clamped = Math.max(0, Math.min(frame, traj.frameCount - 1));+ const base = clamped * K * 3;+ const buf = traj.buffer;+ const names = traj.names;+ const scratch = traj.scratch;++ for (let k = 0; k < K; k++) {+ const off = base + k * 3;+ const slot = scratch[names[k]];+ slot.x = buf[off];+ slot.y = buf[off + 1];+ slot.z = buf[off + 2];+ }+}++function fireSubscribers(traj: TrajectoryData): void {+ for (const cb of traj.subscribers) {+ cb(traj.scratch);+ }+}++export const FileKeypointsSourceProvider: React.FC<{+ recordingId: string | null;+ currentFrameRef: React.MutableRefObject<number>;+ children: React.ReactNode;+}> = ({recordingId, currentFrameRef, children}) => {++ const rawRef = useRef<TrajectoryData>(emptyTrajectory());+ const filteredRef = useRef<TrajectoryData>(emptyTrajectory());++ // Load + decode the parquet whenever recordingId changes.+ useEffect(() => {+ if (!recordingId) {+ rawRef.current = emptyTrajectory();+ filteredRef.current = emptyTrajectory();+ return;+ }++ const controller = new AbortController();++ (async () => {+ try {+ const baseUrl = serverUrls.getHttpUrl();+ const url = `${baseUrl}/freemocap/playback/${encodeURIComponent(recordingId)}/parquet`;++ const t0 = performance.now();+ const resp = await fetch(url, {signal: controller.signal});+ if (!resp.ok) {+ console.warn(`[FileKeypointsSource] parquet fetch failed: ${resp.status}`);+ return;+ }+ const ab = await resp.arrayBuffer();+ const tFetched = performance.now();++ const rows = (await parquetReadObjects({+ file: ab,+ columns: ["frame", "keypoint", "x", "y", "z", "trajectory"],+ })) as ParquetRow[];+ const tDecoded = performance.now();++ if (controller.signal.aborted) return;++ let frameCount = 0;+ for (let i = 0; i < rows.length; i++) {+ const f = rows[i].frame;+ if (f > frameCount) frameCount = f;+ }+ frameCount += 1;++ // Preserve existing subscribers across recording swaps.+ const rawSubs = rawRef.current.subscribers;+ const filteredSubs = filteredRef.current.subscribers;++ const rawTraj = buildTrajectory(rows, TRAJECTORY_PARQUET_VALUE.raw, frameCount);+ const filteredTraj = buildTrajectory(rows, TRAJECTORY_PARQUET_VALUE.filtered, frameCount);+ rawTraj.subscribers = rawSubs;+ filteredTraj.subscribers = filteredSubs;+ rawRef.current = rawTraj;+ filteredRef.current = filteredTraj;++ const tBuilt = performance.now();+ console.info(+ `[FileKeypointsSource] parquet ready: ${rows.length} rows, ${frameCount} frames, ` ++ `raw K=${rawTraj.keypointCount}, filtered K=${filteredTraj.keypointCount} ` ++ `(fetch ${(tFetched - t0) | 0}ms, decode ${(tDecoded - tFetched) | 0}ms, pivot ${(tBuilt - tDecoded) | 0}ms)`+ );+ } catch (err) {+ if ((err as any)?.name !== "AbortError") {+ console.warn("[FileKeypointsSource] load error", err);+ }+ }+ })();++ return () => controller.abort();+ }, [recordingId]);++ // rAF loop: poll the controller's frame ref and push updates when the+ // integer frame changes.+ useEffect(() => {+ let raf = 0;++ const tick = () => {+ const frame = Math.floor(currentFrameRef.current);++ for (const traj of [rawRef.current, filteredRef.current]) {+ if (traj.frameCount === 0) continue;+ if (frame !== traj.lastEmittedFrame) {+ updateScratch(traj, frame);+ traj.lastEmittedFrame = frame;+ fireSubscribers(traj);+ }+ }++ raf = requestAnimationFrame(tick);+ };++ raf = requestAnimationFrame(tick);+ return () => cancelAnimationFrame(raf);+ }, [currentFrameRef]);++ const source = useMemo<KeypointsSource>(() => ({+ subscribeToKeypointsRaw: (cb: KeypointsCallback) => {+ rawRef.current.subscribers.add(cb);+ if (rawRef.current.frameCount > 0 && rawRef.current.lastEmittedFrame >= 0) {+ cb(rawRef.current.scratch);+ }+ return () => { rawRef.current.subscribers.delete(cb); };+ },+ subscribeToKeypointsFiltered: (cb: KeypointsCallback) => {+ filteredRef.current.subscribers.add(cb);+ if (filteredRef.current.frameCount > 0 && filteredRef.current.lastEmittedFrame >= 0) {+ cb(filteredRef.current.scratch);+ }+ return () => { filteredRef.current.subscribers.delete(cb); };+ },+ getLatestKeypointsRaw: () => rawRef.current.scratch,+ }), []);++ return (+ <KeypointsSourceProvider source={source}>+ {children}+ </KeypointsSourceProvider>+ );+};diff --git a/freemocap-ui/src/components/viewport3d/KeypointsSourceContext.tsx b/freemocap-ui/src/components/viewport3d/KeypointsSourceContext.tsxnew file mode 100644index 00000000..1a5f03b8--- /dev/null+++ b/freemocap-ui/src/components/viewport3d/KeypointsSourceContext.tsx@@ -0,0 +1,53 @@+import React, {createContext, useContext, useMemo} from "react";+import {Point3d} from "./helpers/viewport3d-types";+import {useServer} from "@/services";++/**+ * Abstraction over "where do 3D keypoints come from". The Streaming panel+ * feeds the renderers from a live WebSocket (via useServer). The Playback+ * panel feeds them from a cached Float32 buffer driven by the playback slider.+ *+ * Renderers pull whichever source is active via `useKeypointsSource()`.+ * If no provider is mounted, the hook transparently falls back to `useServer()`+ * so existing Streaming code keeps working with zero changes at that layer.+ */+export type KeypointsCallback = (points: Record<string, Point3d>) => void;++export interface KeypointsSource {+ subscribeToKeypointsRaw: (cb: KeypointsCallback) => () => void;+ subscribeToKeypointsFiltered: (cb: KeypointsCallback) => () => void;+ getLatestKeypointsRaw: () => Record<string, Point3d>;+}++const KeypointsSourceContext = createContext<KeypointsSource | null>(null);++export const KeypointsSourceProvider: React.FC<{+ source: KeypointsSource;+ children: React.ReactNode;+}> = ({source, children}) => (+ <KeypointsSourceContext.Provider value={source}>+ {children}+ </KeypointsSourceContext.Provider>+);++/**+ * Returns the active keypoints source. Falls back to the live server if no+ * provider is mounted above the consumer.+ */+export function useKeypointsSource(): KeypointsSource {+ const ctx = useContext(KeypointsSourceContext);+ const server = useServer();++ // Build the live adapter lazily so it doesn't allocate when a provider is present.+ const liveAdapter = useMemo<KeypointsSource>(() => ({+ subscribeToKeypointsRaw: server.subscribeToKeypointsRaw,+ subscribeToKeypointsFiltered: server.subscribeToKeypointsFiltered,+ getLatestKeypointsRaw: server.getLatestKeypointsRaw,+ }), [+ server.subscribeToKeypointsRaw,+ server.subscribeToKeypointsFiltered,+ server.getLatestKeypointsRaw,+ ]);++ return ctx ?? liveAdapter;+}diff --git a/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsx b/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsxindex f3347a8d..998827a3 100644--- a/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsx+++ b/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsx@@ -7,7 +7,7 @@ import { ThreeJsScene } from "./ThreeJsScene";import { ViewportOverlay } from "./scene/ViewportOverlay";import { ViewportStateProvider } from "./scene/ViewportStateContext";import { fitCameraToPoints } from "./scene/SceneCamera";-import { useServer } from "@/services";+import { useKeypointsSource } from "./KeypointsSourceContext";// FreeMoCap's world is Z-up (calibration TOML, triangulated keypoints, etc.).// Set the three.js default up vector before any cameras/objects are created so@@ -17,7 +17,7 @@ Object3D.DEFAULT_UP.set(0, 0, 1);export function ThreeJsCanvas() {const controlsRef = useRef<CameraControlsImpl>(null!);const containerRef = useRef<HTMLDivElement>(null);- const { getLatestKeypointsRaw } = useServer();+ const { getLatestKeypointsRaw } = useKeypointsSource();const handleFit = useCallback(() => {fitCameraToPoints(controlsRef.current, getLatestKeypointsRaw());diff --git a/freemocap-ui/src/components/viewport3d/renderers/ConnectionRenderer.tsx b/freemocap-ui/src/components/viewport3d/renderers/ConnectionRenderer.tsxindex 87b34898..2cecf4a1 100644--- a/freemocap-ui/src/components/viewport3d/renderers/ConnectionRenderer.tsx+++ b/freemocap-ui/src/components/viewport3d/renderers/ConnectionRenderer.tsx@@ -8,6 +8,7 @@ import {import { useFrame } from "@react-three/fiber";import { useServer } from "@/services";import { Point3d } from "@/components/viewport3d";+import { useKeypointsSource } from "../KeypointsSourceContext";import { useViewportState } from "../scene/ViewportStateContext";import {buildSegmentsFromSchema,@@ -26,7 +27,8 @@ import { SKELETON_COLORS } from "../helpers/skeleton-colors";* connections are appended in a reserved tail region of the same buffer.*/export function ConnectionRenderer() {- const { subscribeToKeypointsRaw, getActiveSchema, activeTrackerId, trackerSchemas } = useServer();+ const { getActiveSchema, activeTrackerId, trackerSchemas } = useServer();+ const { subscribeToKeypointsRaw } = useKeypointsSource();const { statsRef } = useViewportState();const calibrationConfig = useAppSelector(selectCalibrationConfig);@@ -90,7 +92,9 @@ export function ConnectionRenderer() {const base = i * 6;const c = colors[i];- if (a && b) {+ const aOk = a && Number.isFinite(a.x) && Number.isFinite(a.y) && Number.isFinite(a.z);+ const bOk = b && Number.isFinite(b.x) && Number.isFinite(b.y) && Number.isFinite(b.z);+ if (aOk && bOk) {positions[base] = a.x; positions[base + 1] = a.y; positions[base + 2] = a.z;positions[base + 3] = b.x; positions[base + 4] = b.y; positions[base + 5] = b.z;vertexColors[base] = c.r; vertexColors[base + 1] = c.g; vertexColors[base + 2] = c.b;diff --git a/freemocap-ui/src/components/viewport3d/renderers/FaceRenderer.tsx b/freemocap-ui/src/components/viewport3d/renderers/FaceRenderer.tsxindex 8d71672d..a4fc9d43 100644--- a/freemocap-ui/src/components/viewport3d/renderers/FaceRenderer.tsx+++ b/freemocap-ui/src/components/viewport3d/renderers/FaceRenderer.tsx@@ -11,11 +11,11 @@ import {Vector3,} from "three";import { useFrame } from "@react-three/fiber";-import { useServer } from "@/services";import { Point3d } from "../helpers/viewport3d-types";import { useViewportState } from "../scene/ViewportStateContext";import { COLORS } from "../helpers/colors";import {FACE_CONTOUR_GROUPS} from "@/components/viewport3d/helpers/face-contours";+import { useKeypointsSource } from "../KeypointsSourceContext";const MAX_FACE_POINTS = 512;const DOT_RADIUS = 0.015;@@ -27,7 +27,7 @@ const FAR_AWAY = new Vector3(1e5, 1e5, 1e5);* Subscribes to raw keypoints and filters for face.* names.*/export function FaceRenderer() {- const { subscribeToKeypointsRaw } = useServer();+ const { subscribeToKeypointsRaw } = useKeypointsSource();const { statsRef } = useViewportState();const dotsRef = useRef<InstancedMesh>(null);diff --git a/freemocap-ui/src/components/viewport3d/renderers/KeypointsRenderer.tsx b/freemocap-ui/src/components/viewport3d/renderers/KeypointsRenderer.tsxindex c563692b..dc7209cf 100644--- a/freemocap-ui/src/components/viewport3d/renderers/KeypointsRenderer.tsx+++ b/freemocap-ui/src/components/viewport3d/renderers/KeypointsRenderer.tsx@@ -13,6 +13,7 @@ import { Point3d } from "../helpers/viewport3d-types";import { useViewportState } from "../scene/ViewportStateContext";import { COLORS } from "../helpers/colors";import { classifyPointName, getPointStyle } from "../helpers/skeleton-config";+import { useKeypointsSource, type KeypointsSource } from "../KeypointsSourceContext";const MAX_POINTS = 1024;const DUMMY = new Object3D();@@ -31,6 +32,7 @@ interface KeypointLayerProps {function KeypointLayer({ subscribeKey, color, radius, statsKey, colorMode = "uniform" }: KeypointLayerProps) {const server = useServer();+ const keypointsSource: KeypointsSource = useKeypointsSource();const { statsRef } = useViewportState();const meshRef = useRef<InstancedMesh>(null);const pointsRef = useRef<Record<string, Point3d>>({}); // plain object, matches wire format@@ -53,7 +55,7 @@ function KeypointLayer({ subscribeKey, color, radius, statsKey, colorMode = "uni}, [server, server.activeTrackerId, server.trackerSchemas]);useEffect(() => {- const subscribeFn = server[subscribeKey];+ const subscribeFn = keypointsSource[subscribeKey];return subscribeFn((pts: Record<string, Point3d>) => {pointsRef.current = pts;dirtyRef.current = true;@@ -66,7 +68,7 @@ function KeypointLayer({ subscribeKey, color, radius, statsKey, colorMode = "uni}}});- }, [server, subscribeKey]);+ }, [keypointsSource, subscribeKey]);useEffect(() => {const mesh = meshRef.current;@@ -91,7 +93,8 @@ function KeypointLayer({ subscribeKey, color, radius, statsKey, colorMode = "unifor (const [name, idx] of nameToIdx.current) {const pt = pts[name];- if (pt) {+ const visible = pt && Number.isFinite(pt.x) && Number.isFinite(pt.y) && Number.isFinite(pt.z);+ if (visible) {const style = colorMode === "byBodyPart" ? getPointStyle(name, colorHints) : null;let scale = style ? style.scale : radius;if (name.includes("hand")) scale *= 0.5; // smaller radius for hand keypointsdiff --git a/freemocap-ui/src/pages/PlaybackPage.tsx b/freemocap-ui/src/pages/PlaybackPage.tsxindex f0ac6c00..c8cdc30a 100644--- a/freemocap-ui/src/pages/PlaybackPage.tsx+++ b/freemocap-ui/src/pages/PlaybackPage.tsx@@ -17,6 +17,7 @@ import type {CameraSettings} from '@/pages/StreamingViewPage';import {SettingsOverlay} from "@/components/ui-components/SettingsOverlay";import {Panel, PanelGroup, PanelResizeHandle} from "react-resizable-panels";import {ThreeJsCanvas} from "@/components/viewport3d/ThreeJsCanvas";+import {FileKeypointsSourceProvider} from "@/components/viewport3d/FileKeypointsSourceProvider";const PlaybackPage: React.FC = () => {const theme = useTheme();@@ -300,7 +301,12 @@ const PlaybackPage: React.FC = () => {<Panel defaultSize={40} minSize={10}><Box sx={{height: '100%'}}>- <ThreeJsCanvas/>+ <FileKeypointsSourceProvider+ recordingId={recordingPath}+ currentFrameRef={controller.currentFrameRef}+ >+ <ThreeJsCanvas/>+ </FileKeypointsSourceProvider></Box></Panel></PanelGroup>diff --git a/freemocap/api/http/playback/playback_router.py b/freemocap/api/http/playback/playback_router.pyindex dcd9e055..578ddd89 100644--- a/freemocap/api/http/playback/playback_router.py+++ b/freemocap/api/http/playback/playback_router.py@@ -445,6 +445,27 @@ def stream_video()+@playback_router.get(+ "/{recording_id}/parquet",+ summary="Serve the recording's freemocap_data_by_frame.parquet",+)+def get_recording_parquet(+ recording_id: str,+ recording_parent_directory: str | None = Query(+ default=None,+ description="Override the default recordings directory",+ ),+) -> FileResponse:+ recording_path = _resolve_recording_path(recording_id, recording_parent_directory)+ p = recording_path / "output_data" / "freemocap_data_by_frame.parquet"+ if not p.is_file():+ raise HTTPException(+ status_code=404,+ detail=f"Parquet file not found: {p}",+ )+ return FileResponse(str(p), media_type="application/octet-stream", filename=p.name)++@playback_router.get("/{recording_id}/timestamps",summary="Get timestamps for all videos in a recording",