Skip to content

Free Motion Capture for Everyone 💀✨

TypeScript

refactor playback

jonmatthisa125861ea98e
diff --git a/freemocap-ui/package-lock.json b/freemocap-ui/package-lock.json
index eb114be3..94b37904 100644
--- a/freemocap-ui/package-lock.json
+++ b/freemocap-ui/package-lock.json
@@ -28,6 +28,7 @@
"country-flag-icons": "^1.6.14",
"d3": "^7.9.0",
"electron-updater": "^6.8.1",
+ "hyparquet": "^1.25.6",
"i18next-browser-languagedetector": "^8.2.1",
"js-yaml": "^4.1.1",
"react-grid-layout": "^2.2.2",
@@ -6534,6 +6535,12 @@
"node": ">= 14"
}
},
+ "node_modules/hyparquet": {
+ "version": "1.25.6",
+ "resolved": "https://registry.npmjs.org/hyparquet/-/hyparquet-1.25.6.tgz",
+ "integrity": "sha512-Q9W5IjkVch3ZMnYd4qFv2q8suu5Jc36yt7J+zUNM9grwnP1S189icp0jdEQKM5HJvQkTVy8NMiQ8n/dM5QAt1A==",
+ "license": "MIT"
+ },
"node_modules/i18next": {
"version": "26.0.3",
"resolved": "https://registry.npmjs.org/i18next/-/i18next-26.0.3.tgz",
diff --git a/freemocap-ui/package.json b/freemocap-ui/package.json
index a1685ed2..9dcf4167 100644
--- a/freemocap-ui/package.json
+++ b/freemocap-ui/package.json
@@ -39,6 +39,7 @@
"country-flag-icons": "^1.6.14",
"d3": "^7.9.0",
"electron-updater": "^6.8.1",
+ "hyparquet": "^1.25.6",
"i18next-browser-languagedetector": "^8.2.1",
"js-yaml": "^4.1.1",
"react-grid-layout": "^2.2.2",
diff --git a/freemocap-ui/src/components/viewport3d/FileKeypointsSourceProvider.tsx b/freemocap-ui/src/components/viewport3d/FileKeypointsSourceProvider.tsx
new file mode 100644
index 00000000..b3ca443f
--- /dev/null
+++ b/freemocap-ui/src/components/viewport3d/FileKeypointsSourceProvider.tsx
@@ -0,0 +1,262 @@
+import React, {useEffect, useMemo, useRef} from "react";
+import {parquetReadObjects} from "hyparquet";
+import {Point3d} from "./helpers/viewport3d-types";
+import {
+ KeypointsCallback,
+ KeypointsSource,
+ KeypointsSourceProvider,
+} from "./KeypointsSourceContext";
+import {serverUrls} from "@/constants/server-urls";
+
+/**
+ * KeypointsSource implementation that reads the recording's long-format
+ * `freemocap_data_by_frame.parquet` directly in the browser (via hyparquet),
+ * pivots long→wide into per-trajectory Float32Array buffers, and emits
+ * per-frame point records in lockstep with the playback slider.
+ *
+ * Design notes:
+ * - Parquet is the canonical freemocap data format; decoding it client-side
+ * keeps the backend simple (one passthrough file endpoint) and avoids
+ * sibling cache artifacts that can drift from the source.
+ * - After decode, each trajectory is packed as Float32Array of length
+ * frameCount * K * 3, giving O(1) per-frame indexing.
+ * - Subscribers receive a pre-allocated scratch record whose Point3d slots
+ * are mutated in place each tick (zero GC pressure).
+ * - Driven by `currentFrameRef` from usePlaybackController, polled via our
+ * own rAF loop. React re-renders are not involved in the hot path.
+ */
+
+type TrajectoryAlias = "raw" | "filtered";
+
+const TRAJECTORY_PARQUET_VALUE: Record<TrajectoryAlias, string> = {
+ raw: "3d_xyz",
+ filtered: "rigid_3d_xyz",
+};
+
+interface TrajectoryData {
+ names: string[];
+ buffer: Float32Array; // length = frameCount * K * 3
+ frameCount: number;
+ keypointCount: number;
+ scratch: Record<string, Point3d>;
+ subscribers: Set<KeypointsCallback>;
+ lastEmittedFrame: number; // -1 means "never emitted"
+}
+
+interface ParquetRow {
+ frame: number;
+ keypoint: string;
+ x: number;
+ y: number;
+ z: number;
+ trajectory: string;
+}
+
+function emptyTrajectory(): TrajectoryData {
+ return {
+ names: [],
+ buffer: new Float32Array(0),
+ frameCount: 0,
+ keypointCount: 0,
+ scratch: {},
+ subscribers: new Set(),
+ lastEmittedFrame: -1,
+ };
+}
+
+function buildTrajectory(
+ rows: ParquetRow[],
+ parquetValue: string,
+ frameCount: number,
+): TrajectoryData {
+ // First pass: collect unique keypoint names for this trajectory.
+ const nameSet = new Set<string>();
+ for (let i = 0; i < rows.length; i++) {
+ const row = rows[i];
+ if (row.trajectory === parquetValue) nameSet.add(String(row.keypoint));
+ }
+ if (nameSet.size === 0) return emptyTrajectory();
+
+ const names = Array.from(nameSet).sort();
+ const nameToIdx = new Map<string, number>();
+ for (let i = 0; i < names.length; i++) nameToIdx.set(names[i], i);
+
+ const K = names.length;
+ const buffer = new Float32Array(frameCount * K * 3);
+ buffer.fill(NaN);
+
+ // Second pass: scatter rows into the packed buffer.
+ for (let i = 0; i < rows.length; i++) {
+ const row = rows[i];
+ if (row.trajectory !== parquetValue) continue;
+ const f = row.frame;
+ if (f < 0 || f >= frameCount) continue;
+ const k = nameToIdx.get(String(row.keypoint));
+ if (k === undefined) continue;
+ const off = (f * K + k) * 3;
+ buffer[off] = row.x;
+ buffer[off + 1] = row.y;
+ buffer[off + 2] = row.z;
+ }
+
+ const scratch: Record<string, Point3d> = {};
+ for (const name of names) scratch[name] = {x: NaN, y: NaN, z: NaN};
+
+ return {
+ names,
+ buffer,
+ frameCount,
+ keypointCount: K,
+ scratch,
+ subscribers: new Set(),
+ lastEmittedFrame: -1,
+ };
+}
+
+function updateScratch(traj: TrajectoryData, frame: number): void {
+ const K = traj.keypointCount;
+ if (K === 0) return;
+ const clamped = Math.max(0, Math.min(frame, traj.frameCount - 1));
+ const base = clamped * K * 3;
+ const buf = traj.buffer;
+ const names = traj.names;
+ const scratch = traj.scratch;
+
+ for (let k = 0; k < K; k++) {
+ const off = base + k * 3;
+ const slot = scratch[names[k]];
+ slot.x = buf[off];
+ slot.y = buf[off + 1];
+ slot.z = buf[off + 2];
+ }
+}
+
+function fireSubscribers(traj: TrajectoryData): void {
+ for (const cb of traj.subscribers) {
+ cb(traj.scratch);
+ }
+}
+
+export const FileKeypointsSourceProvider: React.FC<{
+ recordingId: string | null;
+ currentFrameRef: React.MutableRefObject<number>;
+ children: React.ReactNode;
+}> = ({recordingId, currentFrameRef, children}) => {
+
+ const rawRef = useRef<TrajectoryData>(emptyTrajectory());
+ const filteredRef = useRef<TrajectoryData>(emptyTrajectory());
+
+ // Load + decode the parquet whenever recordingId changes.
+ useEffect(() => {
+ if (!recordingId) {
+ rawRef.current = emptyTrajectory();
+ filteredRef.current = emptyTrajectory();
+ return;
+ }
+
+ const controller = new AbortController();
+
+ (async () => {
+ try {
+ const baseUrl = serverUrls.getHttpUrl();
+ const url = `${baseUrl}/freemocap/playback/${encodeURIComponent(recordingId)}/parquet`;
+
+ const t0 = performance.now();
+ const resp = await fetch(url, {signal: controller.signal});
+ if (!resp.ok) {
+ console.warn(`[FileKeypointsSource] parquet fetch failed: ${resp.status}`);
+ return;
+ }
+ const ab = await resp.arrayBuffer();
+ const tFetched = performance.now();
+
+ const rows = (await parquetReadObjects({
+ file: ab,
+ columns: ["frame", "keypoint", "x", "y", "z", "trajectory"],
+ })) as ParquetRow[];
+ const tDecoded = performance.now();
+
+ if (controller.signal.aborted) return;
+
+ let frameCount = 0;
+ for (let i = 0; i < rows.length; i++) {
+ const f = rows[i].frame;
+ if (f > frameCount) frameCount = f;
+ }
+ frameCount += 1;
+
+ // Preserve existing subscribers across recording swaps.
+ const rawSubs = rawRef.current.subscribers;
+ const filteredSubs = filteredRef.current.subscribers;
+
+ const rawTraj = buildTrajectory(rows, TRAJECTORY_PARQUET_VALUE.raw, frameCount);
+ const filteredTraj = buildTrajectory(rows, TRAJECTORY_PARQUET_VALUE.filtered, frameCount);
+ rawTraj.subscribers = rawSubs;
+ filteredTraj.subscribers = filteredSubs;
+ rawRef.current = rawTraj;
+ filteredRef.current = filteredTraj;
+
+ const tBuilt = performance.now();
+ console.info(
+ `[FileKeypointsSource] parquet ready: ${rows.length} rows, ${frameCount} frames, ` +
+ `raw K=${rawTraj.keypointCount}, filtered K=${filteredTraj.keypointCount} ` +
+ `(fetch ${(tFetched - t0) | 0}ms, decode ${(tDecoded - tFetched) | 0}ms, pivot ${(tBuilt - tDecoded) | 0}ms)`
+ );
+ } catch (err) {
+ if ((err as any)?.name !== "AbortError") {
+ console.warn("[FileKeypointsSource] load error", err);
+ }
+ }
+ })();
+
+ return () => controller.abort();
+ }, [recordingId]);
+
+ // rAF loop: poll the controller's frame ref and push updates when the
+ // integer frame changes.
+ useEffect(() => {
+ let raf = 0;
+
+ const tick = () => {
+ const frame = Math.floor(currentFrameRef.current);
+
+ for (const traj of [rawRef.current, filteredRef.current]) {
+ if (traj.frameCount === 0) continue;
+ if (frame !== traj.lastEmittedFrame) {
+ updateScratch(traj, frame);
+ traj.lastEmittedFrame = frame;
+ fireSubscribers(traj);
+ }
+ }
+
+ raf = requestAnimationFrame(tick);
+ };
+
+ raf = requestAnimationFrame(tick);
+ return () => cancelAnimationFrame(raf);
+ }, [currentFrameRef]);
+
+ const source = useMemo<KeypointsSource>(() => ({
+ subscribeToKeypointsRaw: (cb: KeypointsCallback) => {
+ rawRef.current.subscribers.add(cb);
+ if (rawRef.current.frameCount > 0 && rawRef.current.lastEmittedFrame >= 0) {
+ cb(rawRef.current.scratch);
+ }
+ return () => { rawRef.current.subscribers.delete(cb); };
+ },
+ subscribeToKeypointsFiltered: (cb: KeypointsCallback) => {
+ filteredRef.current.subscribers.add(cb);
+ if (filteredRef.current.frameCount > 0 && filteredRef.current.lastEmittedFrame >= 0) {
+ cb(filteredRef.current.scratch);
+ }
+ return () => { filteredRef.current.subscribers.delete(cb); };
+ },
+ getLatestKeypointsRaw: () => rawRef.current.scratch,
+ }), []);
+
+ return (
+ <KeypointsSourceProvider source={source}>
+ {children}
+ </KeypointsSourceProvider>
+ );
+};
diff --git a/freemocap-ui/src/components/viewport3d/KeypointsSourceContext.tsx b/freemocap-ui/src/components/viewport3d/KeypointsSourceContext.tsx
new file mode 100644
index 00000000..1a5f03b8
--- /dev/null
+++ b/freemocap-ui/src/components/viewport3d/KeypointsSourceContext.tsx
@@ -0,0 +1,53 @@
+import React, {createContext, useContext, useMemo} from "react";
+import {Point3d} from "./helpers/viewport3d-types";
+import {useServer} from "@/services";
+
+/**
+ * Abstraction over "where do 3D keypoints come from". The Streaming panel
+ * feeds the renderers from a live WebSocket (via useServer). The Playback
+ * panel feeds them from a cached Float32 buffer driven by the playback slider.
+ *
+ * Renderers pull whichever source is active via `useKeypointsSource()`.
+ * If no provider is mounted, the hook transparently falls back to `useServer()`
+ * so existing Streaming code keeps working with zero changes at that layer.
+ */
+export type KeypointsCallback = (points: Record<string, Point3d>) => void;
+
+export interface KeypointsSource {
+ subscribeToKeypointsRaw: (cb: KeypointsCallback) => () => void;
+ subscribeToKeypointsFiltered: (cb: KeypointsCallback) => () => void;
+ getLatestKeypointsRaw: () => Record<string, Point3d>;
+}
+
+const KeypointsSourceContext = createContext<KeypointsSource | null>(null);
+
+export const KeypointsSourceProvider: React.FC<{
+ source: KeypointsSource;
+ children: React.ReactNode;
+}> = ({source, children}) => (
+ <KeypointsSourceContext.Provider value={source}>
+ {children}
+ </KeypointsSourceContext.Provider>
+);
+
+/**
+ * Returns the active keypoints source. Falls back to the live server if no
+ * provider is mounted above the consumer.
+ */
+export function useKeypointsSource(): KeypointsSource {
+ const ctx = useContext(KeypointsSourceContext);
+ const server = useServer();
+
+ // Build the live adapter lazily so it doesn't allocate when a provider is present.
+ const liveAdapter = useMemo<KeypointsSource>(() => ({
+ subscribeToKeypointsRaw: server.subscribeToKeypointsRaw,
+ subscribeToKeypointsFiltered: server.subscribeToKeypointsFiltered,
+ getLatestKeypointsRaw: server.getLatestKeypointsRaw,
+ }), [
+ server.subscribeToKeypointsRaw,
+ server.subscribeToKeypointsFiltered,
+ server.getLatestKeypointsRaw,
+ ]);
+
+ return ctx ?? liveAdapter;
+}
diff --git a/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsx b/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsx
index f3347a8d..998827a3 100644
--- a/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsx
+++ b/freemocap-ui/src/components/viewport3d/ThreeJsCanvas.tsx
@@ -7,7 +7,7 @@ import { ThreeJsScene } from "./ThreeJsScene";
import { ViewportOverlay } from "./scene/ViewportOverlay";
import { ViewportStateProvider } from "./scene/ViewportStateContext";
import { fitCameraToPoints } from "./scene/SceneCamera";
-import { useServer } from "@/services";
+import { useKeypointsSource } from "./KeypointsSourceContext";
// FreeMoCap's world is Z-up (calibration TOML, triangulated keypoints, etc.).
// Set the three.js default up vector before any cameras/objects are created so
@@ -17,7 +17,7 @@ Object3D.DEFAULT_UP.set(0, 0, 1);
export function ThreeJsCanvas() {
const controlsRef = useRef<CameraControlsImpl>(null!);
const containerRef = useRef<HTMLDivElement>(null);
- const { getLatestKeypointsRaw } = useServer();
+ const { getLatestKeypointsRaw } = useKeypointsSource();
const handleFit = useCallback(() => {
fitCameraToPoints(controlsRef.current, getLatestKeypointsRaw());
diff --git a/freemocap-ui/src/components/viewport3d/renderers/ConnectionRenderer.tsx b/freemocap-ui/src/components/viewport3d/renderers/ConnectionRenderer.tsx
index 87b34898..2cecf4a1 100644
--- a/freemocap-ui/src/components/viewport3d/renderers/ConnectionRenderer.tsx
+++ b/freemocap-ui/src/components/viewport3d/renderers/ConnectionRenderer.tsx
@@ -8,6 +8,7 @@ import {
import { useFrame } from "@react-three/fiber";
import { useServer } from "@/services";
import { Point3d } from "@/components/viewport3d";
+import { useKeypointsSource } from "../KeypointsSourceContext";
import { useViewportState } from "../scene/ViewportStateContext";
import {
buildSegmentsFromSchema,
@@ -26,7 +27,8 @@ import { SKELETON_COLORS } from "../helpers/skeleton-colors";
* connections are appended in a reserved tail region of the same buffer.
*/
export function ConnectionRenderer() {
- const { subscribeToKeypointsRaw, getActiveSchema, activeTrackerId, trackerSchemas } = useServer();
+ const { getActiveSchema, activeTrackerId, trackerSchemas } = useServer();
+ const { subscribeToKeypointsRaw } = useKeypointsSource();
const { statsRef } = useViewportState();
const calibrationConfig = useAppSelector(selectCalibrationConfig);
@@ -90,7 +92,9 @@ export function ConnectionRenderer() {
const base = i * 6;
const c = colors[i];
- if (a && b) {
+ const aOk = a && Number.isFinite(a.x) && Number.isFinite(a.y) && Number.isFinite(a.z);
+ const bOk = b && Number.isFinite(b.x) && Number.isFinite(b.y) && Number.isFinite(b.z);
+ if (aOk && bOk) {
positions[base] = a.x; positions[base + 1] = a.y; positions[base + 2] = a.z;
positions[base + 3] = b.x; positions[base + 4] = b.y; positions[base + 5] = b.z;
vertexColors[base] = c.r; vertexColors[base + 1] = c.g; vertexColors[base + 2] = c.b;
diff --git a/freemocap-ui/src/components/viewport3d/renderers/FaceRenderer.tsx b/freemocap-ui/src/components/viewport3d/renderers/FaceRenderer.tsx
index 8d71672d..a4fc9d43 100644
--- a/freemocap-ui/src/components/viewport3d/renderers/FaceRenderer.tsx
+++ b/freemocap-ui/src/components/viewport3d/renderers/FaceRenderer.tsx
@@ -11,11 +11,11 @@ import {
Vector3,
} from "three";
import { useFrame } from "@react-three/fiber";
-import { useServer } from "@/services";
import { Point3d } from "../helpers/viewport3d-types";
import { useViewportState } from "../scene/ViewportStateContext";
import { COLORS } from "../helpers/colors";
import {FACE_CONTOUR_GROUPS} from "@/components/viewport3d/helpers/face-contours";
+import { useKeypointsSource } from "../KeypointsSourceContext";
const MAX_FACE_POINTS = 512;
const DOT_RADIUS = 0.015;
@@ -27,7 +27,7 @@ const FAR_AWAY = new Vector3(1e5, 1e5, 1e5);
* Subscribes to raw keypoints and filters for face.* names.
*/
export function FaceRenderer() {
- const { subscribeToKeypointsRaw } = useServer();
+ const { subscribeToKeypointsRaw } = useKeypointsSource();
const { statsRef } = useViewportState();
const dotsRef = useRef<InstancedMesh>(null);
diff --git a/freemocap-ui/src/components/viewport3d/renderers/KeypointsRenderer.tsx b/freemocap-ui/src/components/viewport3d/renderers/KeypointsRenderer.tsx
index c563692b..dc7209cf 100644
--- a/freemocap-ui/src/components/viewport3d/renderers/KeypointsRenderer.tsx
+++ b/freemocap-ui/src/components/viewport3d/renderers/KeypointsRenderer.tsx
@@ -13,6 +13,7 @@ import { Point3d } from "../helpers/viewport3d-types";
import { useViewportState } from "../scene/ViewportStateContext";
import { COLORS } from "../helpers/colors";
import { classifyPointName, getPointStyle } from "../helpers/skeleton-config";
+import { useKeypointsSource, type KeypointsSource } from "../KeypointsSourceContext";
const MAX_POINTS = 1024;
const DUMMY = new Object3D();
@@ -31,6 +32,7 @@ interface KeypointLayerProps {
function KeypointLayer({ subscribeKey, color, radius, statsKey, colorMode = "uniform" }: KeypointLayerProps) {
const server = useServer();
+ const keypointsSource: KeypointsSource = useKeypointsSource();
const { statsRef } = useViewportState();
const meshRef = useRef<InstancedMesh>(null);
const pointsRef = useRef<Record<string, Point3d>>({}); // plain object, matches wire format
@@ -53,7 +55,7 @@ function KeypointLayer({ subscribeKey, color, radius, statsKey, colorMode = "uni
}, [server, server.activeTrackerId, server.trackerSchemas]);
useEffect(() => {
- const subscribeFn = server[subscribeKey];
+ const subscribeFn = keypointsSource[subscribeKey];
return subscribeFn((pts: Record<string, Point3d>) => {
pointsRef.current = pts;
dirtyRef.current = true;
@@ -66,7 +68,7 @@ function KeypointLayer({ subscribeKey, color, radius, statsKey, colorMode = "uni
}
}
});
- }, [server, subscribeKey]);
+ }, [keypointsSource, subscribeKey]);
useEffect(() => {
const mesh = meshRef.current;
@@ -91,7 +93,8 @@ function KeypointLayer({ subscribeKey, color, radius, statsKey, colorMode = "uni
for (const [name, idx] of nameToIdx.current) {
const pt = pts[name];
- if (pt) {
+ const visible = pt && Number.isFinite(pt.x) && Number.isFinite(pt.y) && Number.isFinite(pt.z);
+ if (visible) {
const style = colorMode === "byBodyPart" ? getPointStyle(name, colorHints) : null;
let scale = style ? style.scale : radius;
if (name.includes("hand")) scale *= 0.5; // smaller radius for hand keypoints
diff --git a/freemocap-ui/src/pages/PlaybackPage.tsx b/freemocap-ui/src/pages/PlaybackPage.tsx
index f0ac6c00..c8cdc30a 100644
--- a/freemocap-ui/src/pages/PlaybackPage.tsx
+++ b/freemocap-ui/src/pages/PlaybackPage.tsx
@@ -17,6 +17,7 @@ import type {CameraSettings} from '@/pages/StreamingViewPage';
import {SettingsOverlay} from "@/components/ui-components/SettingsOverlay";
import {Panel, PanelGroup, PanelResizeHandle} from "react-resizable-panels";
import {ThreeJsCanvas} from "@/components/viewport3d/ThreeJsCanvas";
+import {FileKeypointsSourceProvider} from "@/components/viewport3d/FileKeypointsSourceProvider";
const PlaybackPage: React.FC = () => {
const theme = useTheme();
@@ -300,7 +301,12 @@ const PlaybackPage: React.FC = () => {
<Panel defaultSize={40} minSize={10}>
<Box sx={{height: '100%'}}>
- <ThreeJsCanvas/>
+ <FileKeypointsSourceProvider
+ recordingId={recordingPath}
+ currentFrameRef={controller.currentFrameRef}
+ >
+ <ThreeJsCanvas/>
+ </FileKeypointsSourceProvider>
</Box>
</Panel>
</PanelGroup>
diff --git a/freemocap/api/http/playback/playback_router.py b/freemocap/api/http/playback/playback_router.py
index dcd9e055..578ddd89 100644
--- a/freemocap/api/http/playback/playback_router.py
+++ b/freemocap/api/http/playback/playback_router.py
@@ -445,6 +445,27 @@ def stream_video(
)
+@playback_router.get(
+ "/{recording_id}/parquet",
+ summary="Serve the recording's freemocap_data_by_frame.parquet",
+)
+def get_recording_parquet(
+ recording_id: str,
+ recording_parent_directory: str | None = Query(
+ default=None,
+ description="Override the default recordings directory",
+ ),
+) -> FileResponse:
+ recording_path = _resolve_recording_path(recording_id, recording_parent_directory)
+ p = recording_path / "output_data" / "freemocap_data_by_frame.parquet"
+ if not p.is_file():
+ raise HTTPException(
+ status_code=404,
+ detail=f"Parquet file not found: {p}",
+ )
+ return FileResponse(str(p), media_type="application/octet-stream", filename=p.name)
+
+
@playback_router.get(
"/{recording_id}/timestamps",
summary="Get timestamps for all videos in a recording",