Free Motion Capture for Everyone πβ¨
TypeScript
import React, {useEffect, useMemo, useRef} from "react";
import {parquetRead} from "hyparquet";
import type {DecodedArray} from "hyparquet";
import {
KeypointsCallback,
KeypointsFrame,
KeypointsSource,
KeypointsSourceProvider,
} from "./KeypointsSourceContext";
import {serverUrls} from "@/constants/server-urls";
import {VIEWPORT_WORKER} from "./ThreeJsCanvas";
import {useAppDispatch, useAppSelector} from "@/store";
import {fetchPlaybackBundle, selectPlaybackBundle} from "@/store/slices/playback-data/playback-data-slice";
import {calibrationLoadedFromBundle} from "@/store/slices/calibration/calibration-slice";
/**
* KeypointsSource implementation that reads the recording's long-format
* `freemocap_data_by_frame.parquet` directly in the browser (via hyparquet),
* pivots longβwide into per-trajectory Float32Array buffers, and emits
* per-frame KeypointsFrame values in lockstep with the playback slider.
*
* Design notes:
* - Uses parquetRead (columnar API) instead of parquetReadObjects to read column-wise.
* - After decode, each trajectory is packed as Float32Array of length
* frameCount * K * 3, giving O(1) per-frame indexing.
* - Subscribers receive a stable KeypointsFrame whose `scratchInterleaved`
* buffer is mutated in place each tick (zero GC pressure after first frame).
* - Driven by `currentFrameRef` from usePlaybackController, polled via our
* own rAF loop. React re-renders are not involved in the hot path.
*/
const PARQUET_KEYPOINTS_COLUMN = "3d_xyz";
const ParquetColumnNames = {
frame: "frame",
keypoint: "keypoint",
model: "model",
x: "x",
y: "y",
z: "z",
trajectory: "trajectory",
} as const;
interface TrajectoryData {
names: readonly string[];
buffer: Float32Array; // length = frameCount * K * 3 (x, y, z β no vis in parquet)
frameCount: number;
keypointCount: number;
scratchInterleaved: Float32Array; // length = K * 4 (x, y, z, vis) β mutated each tick
scratchFrame: KeypointsFrame; // stable reference β interleaved points at scratchInterleaved
subscribers: Set<KeypointsCallback>;
lastEmittedFrame: number; // -1 means "never emitted"
}
function emptyTrajectory(): TrajectoryData {
const scratchInterleaved = new Float32Array(0);
return {
names: [],
buffer: new Float32Array(0),
frameCount: 0,
keypointCount: 0,
scratchInterleaved,
scratchFrame: { pointNames: [], interleaved: scratchInterleaved },
subscribers: new Set(),
lastEmittedFrame: -1,
};
}
// Helper to check if a value is a BigInt
function isBigInt(value: unknown): value is bigint {
return typeof value === "bigint";
}
// Helper to convert any BigInt-containing array to Float64Array
function convertBigIntArray(arr: DecodedArray): Float64Array | DecodedArray {
if (arr instanceof BigInt64Array || arr instanceof BigUint64Array) {
const out = new Float64Array(arr.length);
for (let i = 0; i < arr.length; i++) {
out[i] = Number(arr[i]);
}
return out;
}
// Check if plain array contains BigInt values
if (Array.isArray(arr) && arr.length > 0 && isBigInt(arr[0])) {
const out = new Float64Array(arr.length);
for (let i = 0; i < arr.length; i++) {
out[i] = Number(arr[i]);
}
return out;
}
return arr;
}
// Merge row-group chunks for a single column into one contiguous array.
function concatDecodedArrays(chunks: DecodedArray[]): DecodedArray {
// First, convert any BigInt arrays to Float64Array
const converted = chunks.map(c => convertBigIntArray(c));
// Even for a single chunk, check if it was a BigInt array (already converted above)
if (converted.length === 1) {
return converted[0];
}
const total = converted.reduce((s, c) => s + c.length, 0);
const first = converted[0];
if (first instanceof Float64Array) {
const out = new Float64Array(total);
let off = 0;
for (const c of converted) { out.set(c as Float64Array, off); off += c.length; }
return out;
}
if (first instanceof Float32Array) {
const out = new Float32Array(total);
let off = 0;
for (const c of converted) { out.set(c as Float32Array, off); off += c.length; }
return out;
}
if (first instanceof Int32Array) {
const out = new Int32Array(total);
let off = 0;
for (const c of converted) { out.set(c as Int32Array, off); off += c.length; }
return out;
}
// String columns or other arrays
const out: unknown[] = [];
for (const c of converted) for (let i = 0; i < c.length; i++) out.push((c as unknown[])[i]);
return out;
}
/**
* Prefix hand keypoint names with their side so they match the holistic schema's
* connection endpoint names. Body and face names are returned unchanged β they
* already carry any needed prefix baked into the landmark name itself.
*
* mediapipe.right_hand + wrist β right_hand_wrist
* mediapipe.left_hand + wrist β left_hand_wrist
* mediapipe.body + nose β nose
*/
function prefixKeypointName(keypoint: string, model: string): string {
const dot = model.indexOf(".");
if (dot === -1) return keypoint;
const aspect = model.slice(dot + 1);
if (aspect === "right_hand" || aspect === "left_hand") {
return `${aspect}_${keypoint}`;
}
return keypoint;
}
function buildTrajectory(
cols: Record<string, DecodedArray>,
parquetValue: string,
frameCount: number,
): TrajectoryData {
// Frame column may be Int32Array or Float64Array (if converted from BigInt)
const frameCol = cols[ParquetColumnNames.frame] as Int32Array | Float64Array;
const keypointCol = cols[ParquetColumnNames.keypoint] as string[];
const modelCol = cols[ParquetColumnNames.model] as string[];
const xCol = cols[ParquetColumnNames.x] as Float64Array;
const yCol = cols[ParquetColumnNames.y] as Float64Array;
const zCol = cols[ParquetColumnNames.z] as Float64Array;
const trajCol = cols[ParquetColumnNames.trajectory] as string[];
const rowCount = frameCol.length;
// First pass: collect unique keypoint names for this trajectory.
// Hand keypoints are prefixed with their side (right_hand_ / left_hand_)
// so left and right don't collide in the flat name set and so the names
// match the holistic schema's connection endpoint names.
const nameSet = new Set<string>();
for (let i = 0; i < rowCount; i++) {
if (trajCol[i] === parquetValue) {
nameSet.add(prefixKeypointName(keypointCol[i], modelCol[i]));
}
}
if (nameSet.size === 0) return emptyTrajectory();
const names = Array.from(nameSet).sort();
const nameToIdx = new Map<string, number>();
for (let i = 0; i < names.length; i++) nameToIdx.set(names[i], i);
const K = names.length;
const buffer = new Float32Array(frameCount * K * 3);
buffer.fill(NaN);
// Second pass: scatter columnar data into the packed buffer.
for (let i = 0; i < rowCount; i++) {
if (trajCol[i] !== parquetValue) continue;
const f = frameCol[i];
if (f < 0 || f >= frameCount) continue;
const k = nameToIdx.get(prefixKeypointName(keypointCol[i], modelCol[i]));
if (k === undefined) continue;
const off = (f * K + k) * 3;
buffer[off] = xCol[i];
buffer[off + 1] = yCol[i];
buffer[off + 2] = zCol[i];
}
const scratchInterleaved = new Float32Array(K * 4);
const scratchFrame: KeypointsFrame = { pointNames: names, interleaved: scratchInterleaved };
return {
names,
buffer,
frameCount,
keypointCount: K,
scratchInterleaved,
scratchFrame,
subscribers: new Set(),
lastEmittedFrame: -1,
};
}
/**
* Rewrite sequential face_XXXX names produced by skellyforge's model_info
* (face_0000β¦face_0135) into the non-contiguous MediaPipe contour-index names
* from the tracker schema (face_0000, face_0007, face_0010β¦). The buffer is
* indexed by position, not name, so only the name list changes β per-frame data
* stays correct. After this, the schema's face-connection endpoint names match
* the trajectory's point names and ConnectionRenderer can draw them.
*/
function remapTrajectoryFaceNames(
traj: TrajectoryData,
contourNames: string[],
): void {
if (traj.frameCount === 0 || contourNames.length === 0) return;
// Collect current face-name slots sorted by their sequential number.
const slots: { idx: number; seq: number }[] = [];
const names = traj.names as string[];
for (let i = 0; i < names.length; i++) {
if (names[i].startsWith("face_")) {
const seq = parseInt(names[i].slice(5), 10);
if (!Number.isNaN(seq)) slots.push({ idx: i, seq });
}
}
slots.sort((a, b) => a.seq - b.seq);
// Replace each sequential slot with the matching contour name.
const newNames = [...names];
for (let i = 0; i < slots.length && i < contourNames.length; i++) {
newNames[slots[i].idx] = contourNames[i];
}
// Mutate in-place so existing refs pick up the new names.
(traj as { names: readonly string[] }).names = newNames;
(traj as { scratchFrame: KeypointsFrame }).scratchFrame = {
pointNames: newNames,
interleaved: traj.scratchInterleaved,
};
}
function updateScratch(traj: TrajectoryData, frame: number): void {
const K = traj.keypointCount;
if (K === 0) return;
const clamped = Math.max(0, Math.min(frame, traj.frameCount - 1));
const base = clamped * K * 3;
const buf = traj.buffer;
const scratch = traj.scratchInterleaved;
for (let k = 0; k < K; k++) {
const srcOff = base + k * 3;
const dstOff = k * 4;
const x = buf[srcOff];
const y = buf[srcOff + 1];
const z = buf[srcOff + 2];
scratch[dstOff] = x;
scratch[dstOff + 1] = y;
scratch[dstOff + 2] = z;
scratch[dstOff + 3] = Number.isFinite(x) && Number.isFinite(y) && Number.isFinite(z) ? 1.0 : 0.0;
}
}
function fireSubscribers(traj: TrajectoryData): void {
for (const cb of traj.subscribers) {
cb(traj.scratchFrame);
}
}
export const FileKeypointsSourceProvider: React.FC<{
recordingId: string | null;
recordingParentDirectory?: string | null;
currentFrameRef: React.RefObject<number>;
children: React.ReactNode;
}> = ({recordingId, recordingParentDirectory, currentFrameRef, children}) => {
const keypointsRef = useRef<TrajectoryData>(emptyTrajectory());
const faceContourNamesRef = useRef<string[]>([]);
const dispatch = useAppDispatch();
// Fetch the playback bundle (calibration + tracker-schema + status) via
// Redux thunk. The `condition` guard prevents duplicate requests when
// StrictMode remounts the component.
useEffect(() => {
if (!recordingId) return;
dispatch(fetchPlaybackBundle({
recordingId,
recordingParentDirectory,
}));
}, [recordingId, recordingParentDirectory, dispatch]);
// When the bundle arrives from Redux, forward calibration + tracker-schema
// to the viewport3d worker and populate the calibration Redux slice so
// ThreeJsCanvas can also consume it.
const bundle = useAppSelector(selectPlaybackBundle(recordingId));
useEffect(() => {
if (!bundle) return;
// Forward tracker schema to the worker so ConnectionRenderer draws skeleton lines.
const schemaName: string = (bundle.trackerSchema as any)?.name || "playback_schema";
VIEWPORT_WORKER.postMessage({
type: "schemaState",
data: {
activeTrackerId: schemaName,
trackerSchemas: {[schemaName]: bundle.trackerSchema},
},
});
// Remap sequential face_XXXX names to contour YAML names so schema
// connections match the trajectory's point names.
const tp: string[] = (bundle.trackerSchema as any)?.tracked_points ?? [];
faceContourNamesRef.current = tp.filter((n: string) => n.startsWith("face_"));
if (faceContourNamesRef.current.length > 0) {
remapTrajectoryFaceNames(keypointsRef.current, faceContourNamesRef.current);
if (keypointsRef.current.lastEmittedFrame >= 0) {
fireSubscribers(keypointsRef.current);
}
}
// Populate the calibration Redux slice so ThreeJsCanvas consumes it.
dispatch(calibrationLoadedFromBundle(bundle.calibration));
}, [bundle, dispatch]);
// Load + decode the parquet whenever recordingId changes.
useEffect(() => {
if (!recordingId) {
keypointsRef.current = emptyTrajectory();
return;
}
const controller = new AbortController();
(async () => {
try {
const baseUrl = serverUrls.getHttpUrl();
const params = new URLSearchParams();
if (recordingParentDirectory) {
params.set('recording_parent_directory', recordingParentDirectory);
}
const qs = params.toString();
const url = `${baseUrl}/freemocap/playback/${encodeURIComponent(recordingId)}/parquet${qs ? `?${qs}` : ''}`;
const t0 = performance.now();
const resp = await fetch(url, {signal: controller.signal});
if (!resp.ok) {
console.warn(`[FileKeypointsSource] parquet fetch failed: ${resp.status}`);
return;
}
const ab = await resp.arrayBuffer();
const tFetched = performance.now();
// Collect column chunks. onChunk fires once per column per row group,
// so large files produce multiple chunks per column that must be merged.
const chunkMap: Record<string, DecodedArray[]> = {};
await parquetRead({
file: ab,
columns: Object.values(ParquetColumnNames),
onChunk: ({columnName, columnData}) => {
(chunkMap[columnName] ??= []).push(columnData);
},
});
const tDecoded = performance.now();
if (controller.signal.aborted) return;
// Merge row-group chunks into single contiguous arrays per column.
const cols: Record<string, DecodedArray> = {};
for (const name of Object.keys(chunkMap)) {
cols[name] = concatDecodedArrays(chunkMap[name]);
}
// Frame column may be Int32Array or Float64Array (if converted from BigInt)
const frameCol = cols[ParquetColumnNames.frame] as Int32Array | Float64Array;
let frameCount = 0;
for (let i = 0; i < frameCol.length; i++) {
if (frameCol[i] > frameCount) frameCount = frameCol[i];
}
frameCount += 1;
// Preserve existing subscribers across recording swaps.
const prevSubs = keypointsRef.current.subscribers;
const traj = buildTrajectory(cols, PARQUET_KEYPOINTS_COLUMN, frameCount);
traj.subscribers = prevSubs;
keypointsRef.current = traj;
// If the playback-bundle schema arrived before the parquet finished
// loading, remap face names now so the first emitted frame already has
// correct contour-index names.
if (faceContourNamesRef.current.length > 0) {
remapTrajectoryFaceNames(traj, faceContourNamesRef.current);
}
const tBuilt = performance.now();
console.info(
`[FileKeypointsSource] parquet ready: ${frameCol.length} rows, ${frameCount} frames, ` +
`K=${traj.keypointCount} ` +
`(fetch ${(tFetched - t0) | 0}ms, decode ${(tDecoded - tFetched) | 0}ms, pivot ${(tBuilt - tDecoded) | 0}ms)`
);
} catch (err) {
if ((err as any)?.name !== "AbortError") {
console.warn("[FileKeypointsSource] load error", err);
}
}
})();
return () => controller.abort();
}, [recordingId, recordingParentDirectory]);
// rAF loop: poll the controller's frame ref and push updates when the
// integer frame changes.
useEffect(() => {
let raf = 0;
const tick = () => {
const frame = Math.floor(currentFrameRef.current);
const traj = keypointsRef.current;
if (traj.frameCount > 0 && frame !== traj.lastEmittedFrame) {
updateScratch(traj, frame);
traj.lastEmittedFrame = frame;
fireSubscribers(traj);
}
raf = requestAnimationFrame(tick);
};
raf = requestAnimationFrame(tick);
return () => cancelAnimationFrame(raf);
}, [currentFrameRef]);
const source = useMemo<KeypointsSource>(() => ({
subscribeToKeypoints: (cb: KeypointsCallback) => {
keypointsRef.current.subscribers.add(cb);
if (keypointsRef.current.frameCount > 0 && keypointsRef.current.lastEmittedFrame >= 0) {
cb(keypointsRef.current.scratchFrame);
}
return () => { keypointsRef.current.subscribers.delete(cb); };
},
subscribeToSkeleton: (_cb: KeypointsCallback) => {
// Playback has no skeleton data β return no-op unsubscribe.
return () => {};
},
getLatestKeypoints: () =>
keypointsRef.current.frameCount > 0 ? keypointsRef.current.scratchFrame : null,
getLatestSkeleton: () => null, // playback has no realtime skeleton fitter
}), []);
return (
<KeypointsSourceProvider source={source}>
{children}
</KeypointsSourceProvider>
);
};