Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

mocap playback kiiinda works

jonmatthis9d980e10bf6b
diff --git a/freemocap-ui/src/components/ui-components/MainNavTabs.tsx b/freemocap-ui/src/components/ui-components/MainNavTabs.tsx
index ca61f1d8..23f3f439 100644
--- a/freemocap-ui/src/components/ui-components/MainNavTabs.tsx
+++ b/freemocap-ui/src/components/ui-components/MainNavTabs.tsx
@@ -6,8 +6,8 @@ import {Tab, Tabs} from "@mui/material";
const NAV_TABS = [
{path: '/welcome', label: 'Home'},
{path: '/streaming', label: 'Streaming'},
- {path: '/browse', label: 'Browse'},
{path: '/playback', label: 'Playback'},
+ {path: '/browse', label: 'Recordings'},
] as const;
export const MainNavTabs = () => {
diff --git a/freemocap-ui/src/components/viewport3d/FileKeypointsSourceProvider.tsx b/freemocap-ui/src/components/viewport3d/FileKeypointsSourceProvider.tsx
index b3ca443f..8867a771 100644
--- a/freemocap-ui/src/components/viewport3d/FileKeypointsSourceProvider.tsx
+++ b/freemocap-ui/src/components/viewport3d/FileKeypointsSourceProvider.tsx
@@ -1,5 +1,6 @@
import React, {useEffect, useMemo, useRef} from "react";
-import {parquetReadObjects} from "hyparquet";
+import {parquetRead} from "hyparquet";
+import type {DecodedArray} from "hyparquet";
import {Point3d} from "./helpers/viewport3d-types";
import {
KeypointsCallback,
@@ -15,9 +16,7 @@ import {serverUrls} from "@/constants/server-urls";
* per-frame point records in lockstep with the playback slider.
*
* Design notes:
- * - Parquet is the canonical freemocap data format; decoding it client-side
- * keeps the backend simple (one passthrough file endpoint) and avoids
- * sibling cache artifacts that can drift from the source.
+ * - Uses parquetRead (columnar API) instead of parquetReadObjects to read column-wise.
* - After decode, each trajectory is packed as Float32Array of length
* frameCount * K * 3, giving O(1) per-frame indexing.
* - Subscribers receive a pre-allocated scratch record whose Point3d slots
@@ -33,6 +32,15 @@ const TRAJECTORY_PARQUET_VALUE: Record<TrajectoryAlias, string> = {
filtered: "rigid_3d_xyz",
};
+const ParquetColumnNames = {
+ frame: "frame",
+ keypoint: "keypoint",
+ x: "x",
+ y: "y",
+ z: "z",
+ trajectory: "trajectory",
+} as const;
+
interface TrajectoryData {
names: string[];
buffer: Float32Array; // length = frameCount * K * 3
@@ -43,15 +51,6 @@ interface TrajectoryData {
lastEmittedFrame: number; // -1 means "never emitted"
}
-interface ParquetRow {
- frame: number;
- keypoint: string;
- x: number;
- y: number;
- z: number;
- trajectory: string;
-}
-
function emptyTrajectory(): TrajectoryData {
return {
names: [],
@@ -64,16 +63,85 @@ function emptyTrajectory(): TrajectoryData {
};
}
+// Helper to check if a value is a BigInt
+function isBigInt(value: unknown): value is bigint {
+ return typeof value === "bigint";
+}
+
+// Helper to convert any BigInt-containing array to Float64Array
+function convertBigIntArray(arr: DecodedArray): Float64Array | DecodedArray {
+ if (arr instanceof BigInt64Array || arr instanceof BigUint64Array) {
+ const out = new Float64Array(arr.length);
+ for (let i = 0; i < arr.length; i++) {
+ out[i] = Number(arr[i]);
+ }
+ return out;
+ }
+ // Check if plain array contains BigInt values
+ if (Array.isArray(arr) && arr.length > 0 && isBigInt(arr[0])) {
+ const out = new Float64Array(arr.length);
+ for (let i = 0; i < arr.length; i++) {
+ out[i] = Number(arr[i]);
+ }
+ return out;
+ }
+ return arr;
+}
+
+// Merge row-group chunks for a single column into one contiguous array.
+function concatDecodedArrays(chunks: DecodedArray[]): DecodedArray {
+ // First, convert any BigInt arrays to Float64Array
+ const converted = chunks.map(c => convertBigIntArray(c));
+
+ // Even for a single chunk, check if it was a BigInt array (already converted above)
+ if (converted.length === 1) {
+ return converted[0];
+ }
+
+ const total = converted.reduce((s, c) => s + c.length, 0);
+ const first = converted[0];
+ if (first instanceof Float64Array) {
+ const out = new Float64Array(total);
+ let off = 0;
+ for (const c of converted) { out.set(c as Float64Array, off); off += c.length; }
+ return out;
+ }
+ if (first instanceof Float32Array) {
+ const out = new Float32Array(total);
+ let off = 0;
+ for (const c of converted) { out.set(c as Float32Array, off); off += c.length; }
+ return out;
+ }
+ if (first instanceof Int32Array) {
+ const out = new Int32Array(total);
+ let off = 0;
+ for (const c of converted) { out.set(c as Int32Array, off); off += c.length; }
+ return out;
+ }
+ // String columns or other arrays
+ const out: unknown[] = [];
+ for (const c of converted) for (let i = 0; i < c.length; i++) out.push((c as unknown[])[i]);
+ return out;
+}
+
function buildTrajectory(
- rows: ParquetRow[],
+ cols: Record<string, DecodedArray>,
parquetValue: string,
frameCount: number,
): TrajectoryData {
+ // Frame column may be Int32Array or Float64Array (if converted from BigInt)
+ const frameCol = cols[ParquetColumnNames.frame] as Int32Array | Float64Array;
+ const keypointCol = cols[ParquetColumnNames.keypoint] as string[];
+ const xCol = cols[ParquetColumnNames.x] as Float64Array;
+ const yCol = cols[ParquetColumnNames.y] as Float64Array;
+ const zCol = cols[ParquetColumnNames.z] as Float64Array;
+ const trajCol = cols[ParquetColumnNames.trajectory] as string[];
+ const rowCount = frameCol.length;
+
// First pass: collect unique keypoint names for this trajectory.
const nameSet = new Set<string>();
- for (let i = 0; i < rows.length; i++) {
- const row = rows[i];
- if (row.trajectory === parquetValue) nameSet.add(String(row.keypoint));
+ for (let i = 0; i < rowCount; i++) {
+ if (trajCol[i] === parquetValue) nameSet.add(keypointCol[i]);
}
if (nameSet.size === 0) return emptyTrajectory();
@@ -85,18 +153,17 @@ function buildTrajectory(
const buffer = new Float32Array(frameCount * K * 3);
buffer.fill(NaN);
- // Second pass: scatter rows into the packed buffer.
- for (let i = 0; i < rows.length; i++) {
- const row = rows[i];
- if (row.trajectory !== parquetValue) continue;
- const f = row.frame;
+ // Second pass: scatter columnar data into the packed buffer.
+ for (let i = 0; i < rowCount; i++) {
+ if (trajCol[i] !== parquetValue) continue;
+ const f = frameCol[i];
if (f < 0 || f >= frameCount) continue;
- const k = nameToIdx.get(String(row.keypoint));
+ const k = nameToIdx.get(keypointCol[i]);
if (k === undefined) continue;
const off = (f * K + k) * 3;
- buffer[off] = row.x;
- buffer[off + 1] = row.y;
- buffer[off + 2] = row.z;
+ buffer[off] = xCol[i];
+ buffer[off + 1] = yCol[i];
+ buffer[off + 2] = zCol[i];
}
const scratch: Record<string, Point3d> = {};
@@ -139,7 +206,7 @@ function fireSubscribers(traj: TrajectoryData): void {
export const FileKeypointsSourceProvider: React.FC<{
recordingId: string | null;
- currentFrameRef: React.MutableRefObject<number>;
+ currentFrameRef: React.RefObject<number>;
children: React.ReactNode;
}> = ({recordingId, currentFrameRef, children}) => {
@@ -170,18 +237,31 @@ export const FileKeypointsSourceProvider: React.FC<{
const ab = await resp.arrayBuffer();
const tFetched = performance.now();
- const rows = (await parquetReadObjects({
+ // Collect column chunks. onChunk fires once per column per row group,
+ // so large files produce multiple chunks per column that must be merged.
+ const chunkMap: Record<string, DecodedArray[]> = {};
+ await parquetRead({
file: ab,
- columns: ["frame", "keypoint", "x", "y", "z", "trajectory"],
- })) as ParquetRow[];
+ columns: Object.values(ParquetColumnNames),
+ onChunk: ({columnName, columnData}) => {
+ (chunkMap[columnName] ??= []).push(columnData);
+ },
+ });
const tDecoded = performance.now();
if (controller.signal.aborted) return;
+ // Merge row-group chunks into single contiguous arrays per column.
+ const cols: Record<string, DecodedArray> = {};
+ for (const name of Object.keys(chunkMap)) {
+ cols[name] = concatDecodedArrays(chunkMap[name]);
+ }
+
+ // Frame column may be Int32Array or Float64Array (if converted from BigInt)
+ const frameCol = cols[ParquetColumnNames.frame] as Int32Array | Float64Array;
let frameCount = 0;
- for (let i = 0; i < rows.length; i++) {
- const f = rows[i].frame;
- if (f > frameCount) frameCount = f;
+ for (let i = 0; i < frameCol.length; i++) {
+ if (frameCol[i] > frameCount) frameCount = frameCol[i];
}
frameCount += 1;
@@ -189,8 +269,8 @@ export const FileKeypointsSourceProvider: React.FC<{
const rawSubs = rawRef.current.subscribers;
const filteredSubs = filteredRef.current.subscribers;
- const rawTraj = buildTrajectory(rows, TRAJECTORY_PARQUET_VALUE.raw, frameCount);
- const filteredTraj = buildTrajectory(rows, TRAJECTORY_PARQUET_VALUE.filtered, frameCount);
+ const rawTraj = buildTrajectory(cols, TRAJECTORY_PARQUET_VALUE.raw, frameCount);
+ const filteredTraj = buildTrajectory(cols, TRAJECTORY_PARQUET_VALUE.filtered, frameCount);
rawTraj.subscribers = rawSubs;
filteredTraj.subscribers = filteredSubs;
rawRef.current = rawTraj;
@@ -198,7 +278,7 @@ export const FileKeypointsSourceProvider: React.FC<{
const tBuilt = performance.now();
console.info(
- `[FileKeypointsSource] parquet ready: ${rows.length} rows, ${frameCount} frames, ` +
+ `[FileKeypointsSource] parquet ready: ${frameCol.length} rows, ${frameCount} frames, ` +
`raw K=${rawTraj.keypointCount}, filtered K=${filteredTraj.keypointCount} ` +
`(fetch ${(tFetched - t0) | 0}ms, decode ${(tDecoded - tFetched) | 0}ms, pivot ${(tBuilt - tDecoded) | 0}ms)`
);
diff --git a/freemocap-ui/src/layout/content/BaseContentRouter.tsx b/freemocap-ui/src/layout/content/BaseContentRouter.tsx
index b8a007bf..c366c7a9 100644
--- a/freemocap-ui/src/layout/content/BaseContentRouter.tsx
+++ b/freemocap-ui/src/layout/content/BaseContentRouter.tsx
@@ -3,7 +3,7 @@ import {Navigate, Route, Routes} from 'react-router-dom';
import WelcomePage from "@/pages/WelcomePage";
import {StreamingViewPage} from "@/pages/StreamingViewPage";
import PlaybackPage from "@/pages/PlaybackPage";
-import BrowsePage from "@/pages/BrowsePage";
+import BrowserPage from "@/pages/BrowserPage";
export const BaseContentRouter: React.FC = () => {
return (
@@ -11,7 +11,7 @@ export const BaseContentRouter: React.FC = () => {
<Route path="/" element={<Navigate to="/welcome" replace/>}/>
<Route path="/welcome" element={<WelcomePage/>}/>
<Route path="/streaming" element={<StreamingViewPage/>}/>
- <Route path="/browse" element={<BrowsePage/>}/>
+ <Route path="/browse" element={<BrowserPage/>}/>
<Route path="/playback" element={<PlaybackPage/>}/>
<Route path="*" element={<Navigate to="/welcome" replace/>}/>
</Routes>
diff --git a/freemocap-ui/src/pages/BrowsePage.tsx b/freemocap-ui/src/pages/BrowserPage.tsx
similarity index 96%
rename from freemocap-ui/src/pages/BrowsePage.tsx
rename to freemocap-ui/src/pages/BrowserPage.tsx
index 11e5bffd..e05051f4 100644
--- a/freemocap-ui/src/pages/BrowsePage.tsx
+++ b/freemocap-ui/src/pages/BrowserPage.tsx
@@ -6,7 +6,7 @@ import ErrorBoundary from '@/components/common/ErrorBoundary';
import {RecordingBrowser} from '@/components/playback/RecordingBrowser';
import {usePlaybackContext} from '@/components/playback/PlaybackContext';
-const BrowsePage: React.FC = () => {
+const BrowserPage: React.FC = () => {
const theme = useTheme();
const navigate = useNavigate();
const ctx = usePlaybackContext();
@@ -44,4 +44,4 @@ const BrowsePage: React.FC = () => {
);
};
-export default BrowsePage;
+export default BrowserPage;
diff --git a/freemocap-ui/src/pages/PlaybackPage.tsx b/freemocap-ui/src/pages/PlaybackPage.tsx
index c8cdc30a..46912a2d 100644
--- a/freemocap-ui/src/pages/PlaybackPage.tsx
+++ b/freemocap-ui/src/pages/PlaybackPage.tsx
@@ -18,6 +18,8 @@ import {SettingsOverlay} from "@/components/ui-components/SettingsOverlay";
import {Panel, PanelGroup, PanelResizeHandle} from "react-resizable-panels";
import {ThreeJsCanvas} from "@/components/viewport3d/ThreeJsCanvas";
import {FileKeypointsSourceProvider} from "@/components/viewport3d/FileKeypointsSourceProvider";
+import {useAppDispatch} from "@/store";
+import {loadCalibrationForRecording} from "@/store/slices/calibration/calibration-thunks";
const PlaybackPage: React.FC = () => {
const theme = useTheme();
@@ -29,6 +31,7 @@ const PlaybackPage: React.FC = () => {
const locationState = location.state as { loadRecordingPath?: string } | null;
const ctx = usePlaybackContext();
+ const dispatch = useAppDispatch();
// Pass initialLoadPath from route state to the context
useEffect(() => {
@@ -36,6 +39,12 @@ const PlaybackPage: React.FC = () => {
if (path) ctx?.setInitialLoadPath(path);
}, [locationState?.loadRecordingPath]);
+ const recordingPathForCalibration = ctx?.recordingPath ?? null;
+ useEffect(() => {
+ if (!recordingPathForCalibration) return;
+ dispatch(loadCalibrationForRecording(recordingPathForCalibration));
+ }, [dispatch, recordingPathForCalibration]);
+
const [settings, setSettings] = useState<CameraSettings>({
columns: null,
show3dView: true,
diff --git a/freemocap-ui/src/store/slices/calibration/calibration-slice.ts b/freemocap-ui/src/store/slices/calibration/calibration-slice.ts
index f025d15b..82a718fa 100644
--- a/freemocap-ui/src/store/slices/calibration/calibration-slice.ts
+++ b/freemocap-ui/src/store/slices/calibration/calibration-slice.ts
@@ -2,6 +2,7 @@ import {createSelector, createSlice, PayloadAction} from '@reduxjs/toolkit';
import {RootState} from '../../types';
import {
calibrateRecording,
+ loadCalibrationForRecording,
loadCalibrationToml,
startCalibrationRecording,
stopCalibrationRecording,
@@ -168,6 +169,16 @@ export const calibrationSlice = createSlice({
state.loadedCalibration = null;
});
+ builder
+ .addCase(loadCalibrationForRecording.fulfilled, (state, action) => {
+ if (action.payload) {
+ state.loadedCalibration = action.payload;
+ }
+ })
+ .addCase(loadCalibrationForRecording.rejected, (state, action) => {
+ console.warn('[calibration] loadCalibrationForRecording rejected:', action.payload);
+ });
+
builder
.addCase(calibrateRecording.pending, (state) => {
state.isLoading = true;
diff --git a/freemocap-ui/src/store/slices/calibration/calibration-thunks.ts b/freemocap-ui/src/store/slices/calibration/calibration-thunks.ts
index 5b21099b..316dccc4 100644
--- a/freemocap-ui/src/store/slices/calibration/calibration-thunks.ts
+++ b/freemocap-ui/src/store/slices/calibration/calibration-thunks.ts
@@ -6,6 +6,31 @@ import {serverUrls} from "@/services";
import {electronIpc} from "@/services/electron-ipc/electron-ipc";
import type {LoadedCalibration} from "./calibration-slice";
+export const loadCalibrationForRecording = createAsyncThunk<
+ LoadedCalibration | null,
+ string,
+ { state: RootState; rejectValue: string }
+>(
+ 'calibration/loadCalibrationForRecording',
+ async (recordingId, { rejectWithValue }) => {
+ try {
+ const url = `${serverUrls.getHttpUrl()}/freemocap/playback/${encodeURIComponent(recordingId)}/calibration`;
+ const resp = await fetch(url);
+ if (resp.status === 404) {
+ console.warn(`[calibration] No calibration found for recording: ${recordingId}`);
+ return null;
+ }
+ if (!resp.ok) {
+ return rejectWithValue(`Calibration fetch failed: ${resp.status}`);
+ }
+ return (await resp.json()) as LoadedCalibration;
+ } catch (error) {
+ const msg = error instanceof Error ? error.message : 'Unknown error';
+ return rejectWithValue(msg);
+ }
+ }
+);
+
export const loadCalibrationToml = createAsyncThunk<
LoadedCalibration | null,
{ path: string; force?: boolean },
diff --git a/freemocap/api/http/playback/playback_router.py b/freemocap/api/http/playback/playback_router.py
index 578ddd89..d071d87f 100644
--- a/freemocap/api/http/playback/playback_router.py
+++ b/freemocap/api/http/playback/playback_router.py
@@ -16,8 +16,9 @@ Endpoints:
import csv
import logging
from pathlib import Path
-from typing import Optional
+from typing import Any, Optional
+import toml
from fastapi import APIRouter, HTTPException, Query
from fastapi.responses import FileResponse
from pydantic import BaseModel
@@ -25,6 +26,7 @@ from pydantic import BaseModel
from freemocap.system.recording_status.recording_status import (
RecordingStatus,
compute_recording_status,
+ _find_calibration_toml,
)
from freemocap.system.default_paths import get_default_freemocap_recordings_path
@@ -466,6 +468,65 @@ def get_recording_parquet(
return FileResponse(str(p), media_type="application/octet-stream", filename=p.name)
+@playback_router.get(
+ "/{recording_id}/calibration",
+ summary="Return parsed calibration data (camera poses) for a recording",
+)
+def get_recording_calibration(
+ recording_id: str,
+ recording_parent_directory: str | None = Query(
+ default=None,
+ description="Override the default recordings directory",
+ ),
+) -> dict[str, Any]:
+ """Parse the recording's calibration TOML and return camera pose data.
+
+ Returns JSON matching the frontend LoadedCalibration shape:
+ { path, mtimeMs, cameras: [...], metadata }
+ Returns 404 if no calibration TOML is present in the recording folder.
+ """
+ recording_path = _resolve_recording_path(recording_id, recording_parent_directory)
+ toml_path = _find_calibration_toml(recording_path)
+ if toml_path is None:
+ raise HTTPException(
+ status_code=404,
+ detail=f"No calibration TOML found in recording: {recording_path}",
+ )
+
+ try:
+ raw = toml_path.read_text(encoding="utf-8")
+ parsed = toml.loads(raw)
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Failed to parse calibration TOML: {e}")
+
+ mtime_ms = toml_path.stat().st_mtime * 1000
+
+ cameras = []
+ for key, val in parsed.items():
+ if key == "metadata" or not isinstance(val, dict):
+ continue
+ if "world_position" not in val or "world_orientation" not in val:
+ continue
+ cameras.append({
+ "id": key,
+ "name": str(val.get("name", key)),
+ "size": val.get("size"),
+ "matrix": val.get("matrix"),
+ "distortions": val.get("distortions"),
+ "rotation": val.get("rotation"),
+ "translation": val.get("translation"),
+ "world_orientation": val["world_orientation"],
+ "world_position": val["world_position"],
+ })
+
+ return {
+ "path": str(toml_path),
+ "mtimeMs": mtime_ms,
+ "cameras": cameras,
+ "metadata": parsed.get("metadata", None),
+ }
+
+
@playback_router.get(
"/{recording_id}/timestamps",
summary="Get timestamps for all videos in a recording",