Free Motion Capture for Everyone πβ¨
TypeScript
mocap playback kiiinda works
jonmatthis9d980e10bf6b
diff --git a/freemocap-ui/src/components/ui-components/MainNavTabs.tsx b/freemocap-ui/src/components/ui-components/MainNavTabs.tsxindex ca61f1d8..23f3f439 100644--- a/freemocap-ui/src/components/ui-components/MainNavTabs.tsx+++ b/freemocap-ui/src/components/ui-components/MainNavTabs.tsx@@ -6,8 +6,8 @@ import {Tab, Tabs} from "@mui/material";const NAV_TABS = [{path: '/welcome', label: 'Home'},{path: '/streaming', label: 'Streaming'},- {path: '/browse', label: 'Browse'},{path: '/playback', label: 'Playback'},+ {path: '/browse', label: 'Recordings'},] as const;export const MainNavTabs = () => {diff --git a/freemocap-ui/src/components/viewport3d/FileKeypointsSourceProvider.tsx b/freemocap-ui/src/components/viewport3d/FileKeypointsSourceProvider.tsxindex b3ca443f..8867a771 100644--- a/freemocap-ui/src/components/viewport3d/FileKeypointsSourceProvider.tsx+++ b/freemocap-ui/src/components/viewport3d/FileKeypointsSourceProvider.tsx@@ -1,5 +1,6 @@import React, {useEffect, useMemo, useRef} from "react";-import {parquetReadObjects} from "hyparquet";+import {parquetRead} from "hyparquet";+import type {DecodedArray} from "hyparquet";import {Point3d} from "./helpers/viewport3d-types";import {KeypointsCallback,@@ -15,9 +16,7 @@ import {serverUrls} from "@/constants/server-urls";* per-frame point records in lockstep with the playback slider.** Design notes:- * - Parquet is the canonical freemocap data format; decoding it client-side- * keeps the backend simple (one passthrough file endpoint) and avoids- * sibling cache artifacts that can drift from the source.+ * - Uses parquetRead (columnar API) instead of parquetReadObjects to read column-wise.* - After decode, each trajectory is packed as Float32Array of length* frameCount * K * 3, giving O(1) per-frame indexing.* - Subscribers receive a pre-allocated scratch record whose Point3d slots@@ -33,6 +32,15 @@ const TRAJECTORY_PARQUET_VALUE: Record<TrajectoryAlias, string> = {filtered: "rigid_3d_xyz",};+const ParquetColumnNames = {+ frame: "frame",+ keypoint: "keypoint",+ x: "x",+ y: "y",+ z: "z",+ trajectory: "trajectory",+} as const;+interface TrajectoryData {names: string[];buffer: Float32Array; // length = frameCount * K * 3@@ -43,15 +51,6 @@ interface TrajectoryData {lastEmittedFrame: number; // -1 means "never emitted"}-interface ParquetRow {- frame: number;- keypoint: string;- x: number;- y: number;- z: number;- trajectory: string;-}-function emptyTrajectory(): TrajectoryData {return {names: [],@@ -64,16 +63,85 @@ function emptyTrajectory(): TrajectoryData {};}+// Helper to check if a value is a BigInt+function isBigInt(value: unknown): value is bigint {+ return typeof value === "bigint";+}++// Helper to convert any BigInt-containing array to Float64Array+function convertBigIntArray(arr: DecodedArray): Float64Array | DecodedArray {+ if (arr instanceof BigInt64Array || arr instanceof BigUint64Array) {+ const out = new Float64Array(arr.length);+ for (let i = 0; i < arr.length; i++) {+ out[i] = Number(arr[i]);+ }+ return out;+ }+ // Check if plain array contains BigInt values+ if (Array.isArray(arr) && arr.length > 0 && isBigInt(arr[0])) {+ const out = new Float64Array(arr.length);+ for (let i = 0; i < arr.length; i++) {+ out[i] = Number(arr[i]);+ }+ return out;+ }+ return arr;+}++// Merge row-group chunks for a single column into one contiguous array.+function concatDecodedArrays(chunks: DecodedArray[]): DecodedArray {+ // First, convert any BigInt arrays to Float64Array+ const converted = chunks.map(c => convertBigIntArray(c));++ // Even for a single chunk, check if it was a BigInt array (already converted above)+ if (converted.length === 1) {+ return converted[0];+ }++ const total = converted.reduce((s, c) => s + c.length, 0);+ const first = converted[0];+ if (first instanceof Float64Array) {+ const out = new Float64Array(total);+ let off = 0;+ for (const c of converted) { out.set(c as Float64Array, off); off += c.length; }+ return out;+ }+ if (first instanceof Float32Array) {+ const out = new Float32Array(total);+ let off = 0;+ for (const c of converted) { out.set(c as Float32Array, off); off += c.length; }+ return out;+ }+ if (first instanceof Int32Array) {+ const out = new Int32Array(total);+ let off = 0;+ for (const c of converted) { out.set(c as Int32Array, off); off += c.length; }+ return out;+ }+ // String columns or other arrays+ const out: unknown[] = [];+ for (const c of converted) for (let i = 0; i < c.length; i++) out.push((c as unknown[])[i]);+ return out;+}+function buildTrajectory(- rows: ParquetRow[],+ cols: Record<string, DecodedArray>,parquetValue: string,frameCount: number,): TrajectoryData {+ // Frame column may be Int32Array or Float64Array (if converted from BigInt)+ const frameCol = cols[ParquetColumnNames.frame] as Int32Array | Float64Array;+ const keypointCol = cols[ParquetColumnNames.keypoint] as string[];+ const xCol = cols[ParquetColumnNames.x] as Float64Array;+ const yCol = cols[ParquetColumnNames.y] as Float64Array;+ const zCol = cols[ParquetColumnNames.z] as Float64Array;+ const trajCol = cols[ParquetColumnNames.trajectory] as string[];+ const rowCount = frameCol.length;+// First pass: collect unique keypoint names for this trajectory.const nameSet = new Set<string>();- for (let i = 0; i < rows.length; i++) {- const row = rows[i];- if (row.trajectory === parquetValue) nameSet.add(String(row.keypoint));+ for (let i = 0; i < rowCount; i++) {+ if (trajCol[i] === parquetValue) nameSet.add(keypointCol[i]);}if (nameSet.size === 0) return emptyTrajectory();@@ -85,18 +153,17 @@ function buildTrajectory(const buffer = new Float32Array(frameCount * K * 3);buffer.fill(NaN);- // Second pass: scatter rows into the packed buffer.- for (let i = 0; i < rows.length; i++) {- const row = rows[i];- if (row.trajectory !== parquetValue) continue;- const f = row.frame;+ // Second pass: scatter columnar data into the packed buffer.+ for (let i = 0; i < rowCount; i++) {+ if (trajCol[i] !== parquetValue) continue;+ const f = frameCol[i];if (f < 0 || f >= frameCount) continue;- const k = nameToIdx.get(String(row.keypoint));+ const k = nameToIdx.get(keypointCol[i]);if (k === undefined) continue;const off = (f * K + k) * 3;- buffer[off] = row.x;- buffer[off + 1] = row.y;- buffer[off + 2] = row.z;+ buffer[off] = xCol[i];+ buffer[off + 1] = yCol[i];+ buffer[off + 2] = zCol[i];}const scratch: Record<string, Point3d> = {};@@ -139,7 +206,7 @@ function fireSubscribers(traj: TrajectoryData): void {export const FileKeypointsSourceProvider: React.FC<{recordingId: string | null;- currentFrameRef: React.MutableRefObject<number>;+ currentFrameRef: React.RefObject<number>;children: React.ReactNode;}> = ({recordingId, currentFrameRef, children}) => {@@ -170,18 +237,31 @@ export const FileKeypointsSourceProvider: React.FC<{const ab = await resp.arrayBuffer();const tFetched = performance.now();- const rows = (await parquetReadObjects({+ // Collect column chunks. onChunk fires once per column per row group,+ // so large files produce multiple chunks per column that must be merged.+ const chunkMap: Record<string, DecodedArray[]> = {};+ await parquetRead({file: ab,- columns: ["frame", "keypoint", "x", "y", "z", "trajectory"],- })) as ParquetRow[];+ columns: Object.values(ParquetColumnNames),+ onChunk: ({columnName, columnData}) => {+ (chunkMap[columnName] ??= []).push(columnData);+ },+ });const tDecoded = performance.now();if (controller.signal.aborted) return;+ // Merge row-group chunks into single contiguous arrays per column.+ const cols: Record<string, DecodedArray> = {};+ for (const name of Object.keys(chunkMap)) {+ cols[name] = concatDecodedArrays(chunkMap[name]);+ }++ // Frame column may be Int32Array or Float64Array (if converted from BigInt)+ const frameCol = cols[ParquetColumnNames.frame] as Int32Array | Float64Array;let frameCount = 0;- for (let i = 0; i < rows.length; i++) {- const f = rows[i].frame;- if (f > frameCount) frameCount = f;+ for (let i = 0; i < frameCol.length; i++) {+ if (frameCol[i] > frameCount) frameCount = frameCol[i];}frameCount += 1;@@ -189,8 +269,8 @@ export const FileKeypointsSourceProvider: React.FC<{const rawSubs = rawRef.current.subscribers;const filteredSubs = filteredRef.current.subscribers;- const rawTraj = buildTrajectory(rows, TRAJECTORY_PARQUET_VALUE.raw, frameCount);- const filteredTraj = buildTrajectory(rows, TRAJECTORY_PARQUET_VALUE.filtered, frameCount);+ const rawTraj = buildTrajectory(cols, TRAJECTORY_PARQUET_VALUE.raw, frameCount);+ const filteredTraj = buildTrajectory(cols, TRAJECTORY_PARQUET_VALUE.filtered, frameCount);rawTraj.subscribers = rawSubs;filteredTraj.subscribers = filteredSubs;rawRef.current = rawTraj;@@ -198,7 +278,7 @@ export const FileKeypointsSourceProvider: React.FC<{const tBuilt = performance.now();console.info(- `[FileKeypointsSource] parquet ready: ${rows.length} rows, ${frameCount} frames, ` ++ `[FileKeypointsSource] parquet ready: ${frameCol.length} rows, ${frameCount} frames, ` +`raw K=${rawTraj.keypointCount}, filtered K=${filteredTraj.keypointCount} ` +`(fetch ${(tFetched - t0) | 0}ms, decode ${(tDecoded - tFetched) | 0}ms, pivot ${(tBuilt - tDecoded) | 0}ms)`);diff --git a/freemocap-ui/src/layout/content/BaseContentRouter.tsx b/freemocap-ui/src/layout/content/BaseContentRouter.tsxindex b8a007bf..c366c7a9 100644--- a/freemocap-ui/src/layout/content/BaseContentRouter.tsx+++ b/freemocap-ui/src/layout/content/BaseContentRouter.tsx@@ -3,7 +3,7 @@ import {Navigate, Route, Routes} from 'react-router-dom';import WelcomePage from "@/pages/WelcomePage";import {StreamingViewPage} from "@/pages/StreamingViewPage";import PlaybackPage from "@/pages/PlaybackPage";-import BrowsePage from "@/pages/BrowsePage";+import BrowserPage from "@/pages/BrowserPage";export const BaseContentRouter: React.FC = () => {return (@@ -11,7 +11,7 @@ export const BaseContentRouter: React.FC = () => {<Route path="/" element={<Navigate to="/welcome" replace/>}/><Route path="/welcome" element={<WelcomePage/>}/><Route path="/streaming" element={<StreamingViewPage/>}/>- <Route path="/browse" element={<BrowsePage/>}/>+ <Route path="/browse" element={<BrowserPage/>}/><Route path="/playback" element={<PlaybackPage/>}/><Route path="*" element={<Navigate to="/welcome" replace/>}/></Routes>diff --git a/freemocap-ui/src/pages/BrowsePage.tsx b/freemocap-ui/src/pages/BrowserPage.tsxsimilarity index 96%rename from freemocap-ui/src/pages/BrowsePage.tsxrename to freemocap-ui/src/pages/BrowserPage.tsxindex 11e5bffd..e05051f4 100644--- a/freemocap-ui/src/pages/BrowsePage.tsx+++ b/freemocap-ui/src/pages/BrowserPage.tsx@@ -6,7 +6,7 @@ import ErrorBoundary from '@/components/common/ErrorBoundary';import {RecordingBrowser} from '@/components/playback/RecordingBrowser';import {usePlaybackContext} from '@/components/playback/PlaybackContext';-const BrowsePage: React.FC = () => {+const BrowserPage: React.FC = () => {const theme = useTheme();const navigate = useNavigate();const ctx = usePlaybackContext();@@ -44,4 +44,4 @@ const BrowsePage: React.FC = () => {);};-export default BrowsePage;+export default BrowserPage;diff --git a/freemocap-ui/src/pages/PlaybackPage.tsx b/freemocap-ui/src/pages/PlaybackPage.tsxindex c8cdc30a..46912a2d 100644--- a/freemocap-ui/src/pages/PlaybackPage.tsx+++ b/freemocap-ui/src/pages/PlaybackPage.tsx@@ -18,6 +18,8 @@ import {SettingsOverlay} from "@/components/ui-components/SettingsOverlay";import {Panel, PanelGroup, PanelResizeHandle} from "react-resizable-panels";import {ThreeJsCanvas} from "@/components/viewport3d/ThreeJsCanvas";import {FileKeypointsSourceProvider} from "@/components/viewport3d/FileKeypointsSourceProvider";+import {useAppDispatch} from "@/store";+import {loadCalibrationForRecording} from "@/store/slices/calibration/calibration-thunks";const PlaybackPage: React.FC = () => {const theme = useTheme();@@ -29,6 +31,7 @@ const PlaybackPage: React.FC = () => {const locationState = location.state as { loadRecordingPath?: string } | null;const ctx = usePlaybackContext();+ const dispatch = useAppDispatch();// Pass initialLoadPath from route state to the contextuseEffect(() => {@@ -36,6 +39,12 @@ const PlaybackPage: React.FC = () => {if (path) ctx?.setInitialLoadPath(path);}, [locationState?.loadRecordingPath]);+ const recordingPathForCalibration = ctx?.recordingPath ?? null;+ useEffect(() => {+ if (!recordingPathForCalibration) return;+ dispatch(loadCalibrationForRecording(recordingPathForCalibration));+ }, [dispatch, recordingPathForCalibration]);+const [settings, setSettings] = useState<CameraSettings>({columns: null,show3dView: true,diff --git a/freemocap-ui/src/store/slices/calibration/calibration-slice.ts b/freemocap-ui/src/store/slices/calibration/calibration-slice.tsindex f025d15b..82a718fa 100644--- a/freemocap-ui/src/store/slices/calibration/calibration-slice.ts+++ b/freemocap-ui/src/store/slices/calibration/calibration-slice.ts@@ -2,6 +2,7 @@ import {createSelector, createSlice, PayloadAction} from '@reduxjs/toolkit';import {RootState} from '../../types';import {calibrateRecording,+ loadCalibrationForRecording,loadCalibrationToml,startCalibrationRecording,stopCalibrationRecording,@@ -168,6 +169,16 @@ export const calibrationSlice = createSlice({state.loadedCalibration = null;});+ builder+ .addCase(loadCalibrationForRecording.fulfilled, (state, action) => {+ if (action.payload) {+ state.loadedCalibration = action.payload;+ }+ })+ .addCase(loadCalibrationForRecording.rejected, (state, action) => {+ console.warn('[calibration] loadCalibrationForRecording rejected:', action.payload);+ });+builder.addCase(calibrateRecording.pending, (state) => {state.isLoading = true;diff --git a/freemocap-ui/src/store/slices/calibration/calibration-thunks.ts b/freemocap-ui/src/store/slices/calibration/calibration-thunks.tsindex 5b21099b..316dccc4 100644--- a/freemocap-ui/src/store/slices/calibration/calibration-thunks.ts+++ b/freemocap-ui/src/store/slices/calibration/calibration-thunks.ts@@ -6,6 +6,31 @@ import {serverUrls} from "@/services";import {electronIpc} from "@/services/electron-ipc/electron-ipc";import type {LoadedCalibration} from "./calibration-slice";+export const loadCalibrationForRecording = createAsyncThunk<+ LoadedCalibration | null,+ string,+ { state: RootState; rejectValue: string }+>(+ 'calibration/loadCalibrationForRecording',+ async (recordingId, { rejectWithValue }) => {+ try {+ const url = `${serverUrls.getHttpUrl()}/freemocap/playback/${encodeURIComponent(recordingId)}/calibration`;+ const resp = await fetch(url);+ if (resp.status === 404) {+ console.warn(`[calibration] No calibration found for recording: ${recordingId}`);+ return null;+ }+ if (!resp.ok) {+ return rejectWithValue(`Calibration fetch failed: ${resp.status}`);+ }+ return (await resp.json()) as LoadedCalibration;+ } catch (error) {+ const msg = error instanceof Error ? error.message : 'Unknown error';+ return rejectWithValue(msg);+ }+ }+);+export const loadCalibrationToml = createAsyncThunk<LoadedCalibration | null,{ path: string; force?: boolean },diff --git a/freemocap/api/http/playback/playback_router.py b/freemocap/api/http/playback/playback_router.pyindex 578ddd89..d071d87f 100644--- a/freemocap/api/http/playback/playback_router.py+++ b/freemocap/api/http/playback/playback_router.py@@ -16,8 +16,9 @@ Endpoints:import csvimport loggingfrom pathlib import Path-from typing import Optional+from typing import Any, Optional+import tomlfrom fastapi import APIRouter, HTTPException, Queryfrom fastapi.responses import FileResponsefrom pydantic import BaseModel@@ -25,6 +26,7 @@ from pydantic import BaseModelfrom freemocap.system.recording_status.recording_status import (RecordingStatus,compute_recording_status,+ _find_calibration_toml,)from freemocap.system.default_paths import get_default_freemocap_recordings_path@@ -466,6 +468,65 @@ def get_recording_parquet(return FileResponse(str(p), media_type="application/octet-stream", filename=p.name)+@playback_router.get(+ "/{recording_id}/calibration",+ summary="Return parsed calibration data (camera poses) for a recording",+)+def get_recording_calibration(+ recording_id: str,+ recording_parent_directory: str | None = Query(+ default=None,+ description="Override the default recordings directory",+ ),+) -> dict[str, Any]:+ """Parse the recording's calibration TOML and return camera pose data.++ Returns JSON matching the frontend LoadedCalibration shape:+ { path, mtimeMs, cameras: [...], metadata }+ Returns 404 if no calibration TOML is present in the recording folder.+ """+ recording_path = _resolve_recording_path(recording_id, recording_parent_directory)+ toml_path = _find_calibration_toml(recording_path)+ if toml_path is None:+ raise HTTPException(+ status_code=404,+ detail=f"No calibration TOML found in recording: {recording_path}",+ )++ try:+ raw = toml_path.read_text(encoding="utf-8")+ parsed = toml.loads(raw)+ except Exception as e:+ raise HTTPException(status_code=500, detail=f"Failed to parse calibration TOML: {e}")++ mtime_ms = toml_path.stat().st_mtime * 1000++ cameras = []+ for key, val in parsed.items():+ if key == "metadata" or not isinstance(val, dict):+ continue+ if "world_position" not in val or "world_orientation" not in val:+ continue+ cameras.append({+ "id": key,+ "name": str(val.get("name", key)),+ "size": val.get("size"),+ "matrix": val.get("matrix"),+ "distortions": val.get("distortions"),+ "rotation": val.get("rotation"),+ "translation": val.get("translation"),+ "world_orientation": val["world_orientation"],+ "world_position": val["world_position"],+ })++ return {+ "path": str(toml_path),+ "mtimeMs": mtime_ms,+ "cameras": cameras,+ "metadata": parsed.get("metadata", None),+ }++@playback_router.get("/{recording_id}/timestamps",summary="Get timestamps for all videos in a recording",