Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

add overlay view and clean up sidebar stuff

jonmatthisf001843cb4af
diff --git a/freemocap-ui/src/components/pipeline-config/PipelineConfigTree.tsx b/freemocap-ui/src/components/pipeline-config/PipelineConfigTree.tsx
index 7463499e..412f3834 100644
--- a/freemocap-ui/src/components/pipeline-config/PipelineConfigTree.tsx
+++ b/freemocap-ui/src/components/pipeline-config/PipelineConfigTree.tsx
@@ -24,9 +24,6 @@ interface PipelineConfigTreeProps {
onFilterToggle: (checked: boolean) => void;
rigidBodyEnabled: boolean;
onRigidBodyToggle: (checked: boolean) => void;
- // Disable all controls (e.g. pipeline not connected)
- disabled?: boolean;
- disabledReason?: string;
}
export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({
@@ -41,8 +38,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({
onFilterToggle,
rigidBodyEnabled,
onRigidBodyToggle,
- disabled = false,
- disabledReason,
}) => {
const theme = useTheme();
const [expandedItems, setExpandedItems] = useState<string[]>([]);
@@ -79,8 +74,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({
label="2D Tracking"
checked={tracking2dEnabled}
onToggle={handleToggle2dTracking}
- disabled={disabled}
- disabledReason={disabledReason}
isExpanded={isExpanded("2d-tracking")}
summaryWhenCollapsed={
[charucoEnabled && "Charuco", skeletonEnabled && "Skeleton"]
@@ -94,8 +87,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({
label="Charuco"
checked={charucoEnabled}
onToggle={onCharucoToggle}
- disabled={disabled}
- disabledReason={disabledReason}
isExpanded={isExpanded("2d-charuco")}
>
<Box sx={{p: 1, pl: 2}}>
@@ -111,8 +102,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({
label="Skeleton"
checked={skeletonEnabled}
onToggle={onSkeletonToggle}
- disabled={disabled}
- disabledReason={disabledReason}
isExpanded={isExpanded("2d-skeleton")}
summaryWhenCollapsed={context === "realtime" ? "Realtime preset" : "Posthoc preset"}
>
@@ -128,8 +117,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({
label="3D Reconstruction"
checked={reconstruction3dEnabled}
onToggle={handleToggle3dReconstruction}
- disabled={disabled}
- disabledReason={disabledReason}
isExpanded={isExpanded("3d-reconstruction")}
summaryWhenCollapsed={
[triangulateEnabled && "Triangulate", filterEnabled && "Filter", rigidBodyEnabled && "Skeleton"]
@@ -143,8 +130,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({
label="Triangulate"
checked={triangulateEnabled}
onToggle={onTriangulateToggle}
- disabled={disabled}
- disabledReason={disabledReason}
isExpanded={isExpanded("3d-triangulate")}
>
<Box sx={{p: 1, pl: 2}}>
@@ -160,8 +145,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({
label="Filter"
checked={filterEnabled}
onToggle={onFilterToggle}
- disabled={disabled}
- disabledReason={disabledReason}
isExpanded={isExpanded("3d-filter")}
summaryWhenCollapsed="One Euro, FABRIK"
>
@@ -176,8 +159,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({
label="Skeleton"
checked={rigidBodyEnabled}
onToggle={onRigidBodyToggle}
- disabled={disabled}
- disabledReason={disabledReason}
isExpanded={isExpanded("3d-skeleton")}
>
<Box sx={{p: 1, pl: 2}}>
diff --git a/freemocap-ui/src/components/processing-pipeline-panel/ProcessingPipelinePanel.tsx b/freemocap-ui/src/components/processing-pipeline-panel/ProcessingPipelinePanel.tsx
index 0bc5e19e..f28fe7b2 100644
--- a/freemocap-ui/src/components/processing-pipeline-panel/ProcessingPipelinePanel.tsx
+++ b/freemocap-ui/src/components/processing-pipeline-panel/ProcessingPipelinePanel.tsx
@@ -14,8 +14,7 @@ export const ProcessingPipelinePanel: React.FC = () => {
const backendPipeline = useAppSelector(selectBackendPipeline);
const pipelineConfig = backendPipeline?.config;
- const isPipelineConnected = backendPipeline?.is_connected ?? false;
- const canToggle = isConnected && isPipelineConnected;
+
// 3D Reconstruction toggles β€” local state until backend support is added
const [triangulateEnabled, setTriangulateEnabled] = useState(false);
@@ -38,11 +37,7 @@ export const ProcessingPipelinePanel: React.FC = () => {
defaultExpanded={false}
>
<Box sx={{p: 2}}>
- {!isPipelineConnected && (
- <Typography variant="caption" color="text.secondary" sx={{fontStyle: "italic", mb: 1, display: "block"}}>
- Connect a pipeline to enable detection toggles.
- </Typography>
- )}
+
<PipelineConfigTree
context="realtime"
charucoEnabled={pipelineConfig?.calibration_detection_enabled ?? true}
@@ -55,7 +50,6 @@ export const ProcessingPipelinePanel: React.FC = () => {
onFilterToggle={setFilterEnabled}
rigidBodyEnabled={rigidBodyEnabled}
onRigidBodyToggle={setRigidBodyEnabled}
- disabled={!canToggle}
disabledReason="Connect a pipeline to enable detection toggles"
/>
</Box>
diff --git a/freemocap-ui/src/services/server/ServerContextProvider.tsx b/freemocap-ui/src/services/server/ServerContextProvider.tsx
index bc4ea8e7..f1e95842 100644
--- a/freemocap-ui/src/services/server/ServerContextProvider.tsx
+++ b/freemocap-ui/src/services/server/ServerContextProvider.tsx
@@ -8,6 +8,13 @@ import { serverUrls } from "@/services";
import {DetailedFramerate, FramerateStore} from "@/services/server/server-helpers/framerate-store";
import {LogStore, LogRecord} from "@/services/server/server-helpers/log-store";
import { Point3d, RigidBodyPose } from "@/components/viewport3d/viewport3d-types";
+import { OverlayManager } from "@/services/server/server-helpers/image-overlay/overlay-renderer-factory";
+import { CharucoObservation } from "@/services/server/server-helpers/image-overlay/charuco-types";
+import { MediapipeObservation } from "@/services/server/server-helpers/image-overlay/mediapipe-types";
+import {
+ isCharucoOverlayDataMessage,
+ isMediapipeOverlayDataMessage
+} from "@/services/server/server-helpers/websocket-message-types";
interface ServerContextValue {
isConnected: boolean;
@@ -81,6 +88,11 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ child
const canvasManagerRef = useRef<CanvasManager | null>(null);
const framerateStoreRef = useRef<FramerateStore>(new FramerateStore());
const logStoreRef = useRef<LogStore>(new LogStore());
+ const overlayManagerRef = useRef<OverlayManager | null>(null);
+
+ // Overlay data refs - NO REACT STATE to avoid re-renders!
+ const latestCharucoRef = useRef<Map<string, CharucoObservation>>(new Map());
+ const latestMediapipeRef = useRef<Map<string, MediapipeObservation>>(new Map());
// Latest server-side (backend) FPS stored in a ref for non-reactive access
const serverFpsRef = useRef<number | null>(null);
@@ -116,6 +128,7 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ child
});
frameProcessorRef.current = new FrameProcessor();
canvasManagerRef.current = new CanvasManager();
+ overlayManagerRef.current = new OverlayManager();
return () => {
if (wsConnectionRef.current) {
@@ -127,6 +140,7 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ child
if (frameProcessorRef.current) {
frameProcessorRef.current.reset();
}
+ overlayManagerRef.current?.clearAll();
};
}, []);
@@ -149,20 +163,22 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ child
framerateStoreRef.current.clear();
trackedPointsRef.current = new Map();
rigidBodiesRef.current = new Map();
+ latestCharucoRef.current.clear();
+ latestMediapipeRef.current.clear();
+ overlayManagerRef.current?.clearAll();
setConnectedCameraIds([]);
}
};
// Process a decoded frame result: update camera list, dispatch to workers, send ack.
- const dispatchFrames = (
+ const dispatchFrames = async (
result: Awaited<ReturnType<FrameProcessor['processFramePayload']>>
- ): void => {
+ ): Promise<void> => {
if (!result) return;
const { frames, cameraIds, frameNumbers } = result;
// Only allocate a new sorted array if the camera set actually changed.
- // Compare against the cached ref to avoid Array.from().sort() on every frame.
const lastIds = lastCameraIdsRef.current;
let cameraListChanged = lastIds.length !== cameraIds.size;
if (!cameraListChanged) {
@@ -183,6 +199,8 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ child
const removedCameras = prevIds.filter(id => !cameraIds.has(id));
for (const cameraId of removedCameras) {
canvasManagerRef.current?.terminateWorker(cameraId);
+ latestCharucoRef.current.delete(cameraId);
+ latestMediapipeRef.current.delete(cameraId);
}
return newIds;
}
@@ -190,10 +208,27 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ child
});
}
+ // Composite overlays onto frames BEFORE sending to canvas workers
+ const overlayManager = overlayManagerRef.current!;
for (const frameData of frames) {
+ const charucoObs = latestCharucoRef.current.get(frameData.cameraId) ?? null;
+ const mediapipeObs = latestMediapipeRef.current.get(frameData.cameraId) ?? null;
+
+ let compositeBitmap: ImageBitmap;
+ if (charucoObs || mediapipeObs) {
+ compositeBitmap = await overlayManager.processFrame(
+ frameData.cameraId,
+ frameData.bitmap,
+ charucoObs,
+ mediapipeObs,
+ );
+ } else {
+ compositeBitmap = frameData.bitmap;
+ }
+
canvasManagerRef.current!.sendFrameToWorker(
frameData.cameraId,
- frameData.bitmap
+ compositeBitmap,
);
}
@@ -213,7 +248,7 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ child
processingFrameRef.current = true;
try {
const result = await frameProcessorRef.current!.processFramePayload(payload);
- dispatchFrames(result);
+ await dispatchFrames(result);
} catch (error) {
console.error('Error processing frame:', error);
} finally {
@@ -251,6 +286,18 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ child
framerateStoreRef.current.updateBackend(jsonData.backend_framerate);
framerateStoreRef.current.updateFrontend(jsonData.frontend_framerate);
}
+ // Handle charuco overlay data
+ else if (isCharucoOverlayDataMessage(jsonData)) {
+ for (const [cameraId, observation] of Object.entries(jsonData)) {
+ latestCharucoRef.current.set(cameraId, observation as CharucoObservation);
+ }
+ }
+ // Handle mediapipe overlay data
+ else if (isMediapipeOverlayDataMessage(jsonData)) {
+ for (const [cameraId, observation] of Object.entries(jsonData)) {
+ latestMediapipeRef.current.set(cameraId, observation as MediapipeObservation);
+ }
+ }
// Handle tracked 3D points
else if (jsonData.tracked_points3d && typeof jsonData.tracked_points3d === 'object') {
const pointsMap = new Map<string, Point3d>();
@@ -387,4 +434,4 @@ export const useServer = (): ServerContextValue => {
const context = useContext(ServerContext);
if (!context) throw new Error('useServer must be used within ServerContextProvider');
return context;
-};
+};
\ No newline at end of file
diff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/charuco-overlay-renderer.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/charuco-overlay-renderer.ts
new file mode 100644
index 00000000..a9fd2b27
--- /dev/null
+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/charuco-overlay-renderer.ts
@@ -0,0 +1,184 @@
+// charuco-overlay-renderer.ts
+
+import {
+ BaseOverlayRenderer,
+ DrawStyle,
+ Point2D
+} from "@/services/server/server-helpers/image-overlay/image-overlay-system";
+import {
+ ArucoMarker,
+ CharucoObservation,
+ CharucoPoint
+} from "@/services/server/server-helpers/image-overlay/charuco-types";
+
+export class CharucoOverlayRenderer extends BaseOverlayRenderer {
+ // Charuco-specific styles
+ private readonly charucoStyle: DrawStyle = {
+ pointColor: '#00FF00',
+ pointStroke: '#009600',
+ pointRadius: 5,
+ lineColor: '#00FF00',
+ lineWidth: 2,
+ labelColor: '#00FF00',
+ labelStroke: this.TEXT_STROKE,
+ labelFontSize: 10,
+ showLabels: true,
+ };
+
+ private readonly arucoStyle: DrawStyle = {
+ pointColor: '#FF6400',
+ pointStroke: '#C85000',
+ pointRadius: 4,
+ lineColor: '#FF6400',
+ lineWidth: 3,
+ labelColor: this.TEXT_COLOR,
+ labelStroke: '#FF6400',
+ labelFontSize: 14,
+ showLabels: true,
+ };
+
+ /**
+ * Composite charuco overlay onto frame
+ */
+ public async compositeFrame(
+ sourceBitmap: ImageBitmap,
+ observation: CharucoObservation | null,
+ ): Promise<ImageBitmap> {
+ this.prepareCanvas(sourceBitmap);
+
+ if (observation) {
+ this.drawCharucoOverlay(observation);
+ }
+
+ return this.createBitmap(sourceBitmap);
+ }
+
+ private drawCharucoOverlay(observation: CharucoObservation): void {
+ this.ctx.save();
+
+ // Draw ArUco markers first (behind Charuco corners)
+ this.drawArucoMarkers(observation.aruco_markers);
+
+ // Draw Charuco corners
+ this.drawCharucoCorners(observation.charuco_corners);
+
+ // Draw segments if model info is available
+ if (this.modelInfo) {
+ this.drawCharucoSegments(observation.charuco_corners);
+ }
+
+ // Draw info overlay
+ this.drawCharucoInfo(observation);
+
+ this.ctx.restore();
+ }
+
+ private drawCharucoCorners(corners: CharucoPoint[]): void {
+ if (corners.length === 0) return;
+
+ // Convert CharucoPoint[] to Point2D[]
+ const points: Point2D[] = corners.map(corner => ({
+ x: corner.x,
+ y: corner.y,
+ id: corner.id,
+ }));
+
+ this.drawPoints(points, this.charucoStyle);
+ }
+
+ private drawCharucoSegments(corners: CharucoPoint[]): void {
+ if (!this.modelInfo || corners.length === 0) return;
+
+ // Create a map of corner ID to Point2D
+ const pointMap = new Map<string | number, Point2D>();
+ for (const corner of corners) {
+ pointMap.set(corner.id.toString(), {
+ x: corner.x,
+ y: corner.y,
+ id: corner.id,
+ });
+ }
+
+ // Get segments from model info
+ const bodyAspect = this.modelInfo.aspects['body'];
+ if (bodyAspect?.segment_connections) {
+ this.drawSegments(pointMap, bodyAspect.segment_connections, {
+ ...this.charucoStyle,
+ lineColor: 'rgba(0, 255, 0, 0.5)', // Semi-transparent green
+ lineWidth: 1,
+ });
+ }
+ }
+
+ private drawArucoMarkers(markers: ArucoMarker[]): void {
+ if (markers.length === 0) return;
+
+ for (const marker of markers) {
+ this.drawPolygon(
+ marker.corners,
+ this.arucoStyle,
+ true,
+ marker.id.toString()
+ );
+ }
+ }
+
+ private drawCharucoInfo(observation: CharucoObservation): void {
+ const { metadata, frame_number } = observation;
+
+ // Build status info
+ let statusText = '';
+ let statusColor = '';
+ if (metadata.n_charuco_detected === 0 && metadata.n_aruco_detected === 0) {
+ statusText = '⚠ NO BOARD DETECTED';
+ statusColor = '#FF4444';
+ } else if (metadata.n_charuco_detected < 4) {
+ statusText = '⚠ INSUFFICIENT CORNERS';
+ statusColor = '#FFAA00';
+ } else {
+ statusText = 'βœ” BOARD DETECTED';
+ statusColor = '#00FF00';
+ }
+
+ const additionalInfo = [
+ `Charuco: ${metadata.n_charuco_detected}/${metadata.n_charuco_total} | ` +
+ `ArUco: ${metadata.n_aruco_detected}/${metadata.n_aruco_total}` +
+ (metadata.has_pose ? ' | POSE βœ”' : ''),
+ statusText,
+ ];
+
+ // Draw background and info
+ this.ctx.fillStyle = this.INFO_BG;
+ this.ctx.fillRect(5, 5, 500, 85);
+
+ this.drawText(
+ `Frame: ${frame_number}`,
+ 10,
+ 25,
+ 14,
+ this.TEXT_COLOR,
+ this.TEXT_STROKE,
+ 2
+ );
+
+ this.drawText(
+ additionalInfo[0],
+ 10,
+ 50,
+ 14,
+ this.TEXT_COLOR,
+ this.TEXT_STROKE,
+ 2
+ );
+
+ this.drawText(
+ additionalInfo[1],
+ 10,
+ 75,
+ 16,
+ statusColor,
+ this.TEXT_STROKE,
+ 2
+ );
+ }
+}
\ No newline at end of file
diff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/charuco-types.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/charuco-types.ts
new file mode 100644
index 00000000..f3edadbe
--- /dev/null
+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/charuco-types.ts
@@ -0,0 +1,43 @@
+import {z} from 'zod';
+
+// Zod schemas for CharucoObservation data
+export const CharucoPointSchema = z.object({
+ id: z.number(),
+ x: z.number(),
+ y: z.number(),
+});
+
+export const ArucoMarkerSchema = z.object({
+ id: z.number(),
+ corners: z.array(z.tuple([z.number(), z.number()])).length(4),
+});
+
+export const CharucoMetadataSchema = z.object({
+ n_charuco_detected: z.number(),
+ n_charuco_total: z.number(),
+ n_aruco_detected: z.number(),
+ n_aruco_total: z.number(),
+ has_pose: z.boolean(),
+ image_width: z.number(),
+ image_height: z.number(),
+});
+
+// Schema for a single camera's charuco observation
+export const CharucoOverlaySchema = z.object({
+ message_type: z.literal("charuco_overlay"),
+ camera_id: z.string(),
+ frame_number: z.number(),
+ charuco_corners: z.array(CharucoPointSchema),
+ aruco_markers: z.array(ArucoMarkerSchema),
+ metadata: CharucoMetadataSchema,
+});
+
+// Schema for the websocket message containing multiple camera observations
+// Structure: { "camera_1": {...}, "camera_2": {...}, ... }
+export const CharucoOverlayDataMessageSchema = z.record(z.string(), CharucoOverlaySchema);
+
+// Type exports
+export type CharucoPoint = z.infer<typeof CharucoPointSchema>;
+export type ArucoMarker = z.infer<typeof ArucoMarkerSchema>;
+export type CharucoObservation = z.infer<typeof CharucoOverlaySchema>;
+export type CharucoOverlayDataMessage = z.infer<typeof CharucoOverlayDataMessageSchema>;
\ No newline at end of file
diff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/image-overlay-system.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/image-overlay-system.ts
new file mode 100644
index 00000000..f0fe2b21
--- /dev/null
+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/image-overlay-system.ts
@@ -0,0 +1,303 @@
+// base-overlay-renderer.ts
+import { z } from 'zod';
+
+// Generic types for any observation
+export interface Point2D {
+ x: number;
+ y: number;
+ id?: string | number;
+ visibility?: number; // 0-1, for confidence/visibility
+}
+
+export interface Segment {
+ proximal: string | number;
+ distal: string | number;
+}
+
+export interface ModelInfo {
+ name: string;
+ tracker_name: string;
+ aspects: Record<string, AspectInfo>;
+ order: string[];
+}
+
+export interface AspectInfo {
+ tracked_points: {
+ type: 'list' | 'generated';
+ names?: string[];
+ convention?: string;
+ count?: number;
+ };
+ segment_connections?: Record<string, Segment>;
+ joint_hierarchy?: Record<string, string[]>;
+}
+
+export interface DrawStyle {
+ pointColor: string;
+ pointStroke: string;
+ pointRadius: number;
+ lineColor: string;
+ lineWidth: number;
+ labelColor: string;
+ labelStroke: string;
+ labelFontSize: number;
+ showLabels: boolean;
+}
+
+export interface ObservationMetadata {
+ frame_number: number;
+ [key: string]: any; // Allow additional metadata
+}
+
+export abstract class BaseOverlayRenderer {
+ protected offscreenCanvas: OffscreenCanvas;
+ protected ctx: OffscreenCanvasRenderingContext2D;
+ protected modelInfo: ModelInfo | null = null;
+
+ // Common style constants
+ protected readonly INFO_BG = 'rgba(0, 0, 0, 0.7)';
+ protected readonly TEXT_COLOR = '#ccc';
+ protected readonly TEXT_STROKE = '#111';
+
+ constructor() {
+ this.offscreenCanvas = new OffscreenCanvas(1, 1);
+ const ctx = this.offscreenCanvas.getContext('2d', {
+ alpha: true,
+ desynchronized: true,
+ });
+
+ if (!ctx) {
+ throw new Error('Failed to get 2D context for overlay renderer');
+ }
+
+ this.ctx = ctx;
+ }
+
+ /**
+ * Set the model info for this renderer
+ */
+ public setModelInfo(modelInfo: ModelInfo): void {
+ this.modelInfo = modelInfo;
+ }
+
+ /**
+ * Main composite method - to be implemented by specific renderers
+ */
+ public abstract compositeFrame(
+ sourceBitmap: ImageBitmap,
+ observation: any | null,
+ ): Promise<ImageBitmap>;
+
+ /**
+ * Resize canvas if needed and draw source image
+ */
+ protected prepareCanvas(sourceBitmap: ImageBitmap): void {
+ const { width, height } = sourceBitmap;
+
+ if (this.offscreenCanvas.width !== width || this.offscreenCanvas.height !== height) {
+ this.offscreenCanvas.width = width;
+ this.offscreenCanvas.height = height;
+ }
+
+ this.ctx.clearRect(0, 0, width, height);
+ this.ctx.drawImage(sourceBitmap, 0, 0);
+ }
+
+ /**
+ * Generic method to draw points
+ */
+ protected drawPoints(
+ points: Point2D[],
+ style: DrawStyle
+ ): void {
+ for (const point of points) {
+ // Skip invalid points
+ if (!this.isValidPoint(point)) continue;
+
+ // Optional visibility threshold
+ if (point.visibility !== undefined && point.visibility < 0.5) continue;
+
+ // Draw filled circle
+ this.ctx.fillStyle = style.pointColor;
+ this.ctx.beginPath();
+ this.ctx.arc(point.x, point.y, style.pointRadius, 0, Math.PI * 2);
+ this.ctx.fill();
+
+ // Draw stroke
+ this.ctx.strokeStyle = style.pointStroke;
+ this.ctx.lineWidth = 2;
+ this.ctx.stroke();
+
+ // Draw label if enabled
+ if (style.showLabels && point.id !== undefined) {
+ this.drawText(
+ point.id.toString(),
+ point.x + style.pointRadius + 3,
+ point.y - style.pointRadius - 3,
+ style.labelFontSize,
+ style.labelColor,
+ style.labelStroke,
+ 2
+ );
+ }
+ }
+ }
+
+ /**
+ * Generic method to draw segments/connections between points
+ */
+ protected drawSegments(
+ points: Map<string | number, Point2D>,
+ segments: Record<string, Segment>,
+ style: DrawStyle
+ ): void {
+ for (const [_, segment] of Object.entries(segments)) {
+ const proximalPoint = points.get(segment.proximal);
+ const distalPoint = points.get(segment.distal);
+
+ if (!proximalPoint || !distalPoint) continue;
+ if (!this.isValidPoint(proximalPoint) || !this.isValidPoint(distalPoint)) continue;
+
+ // Draw line
+ this.ctx.strokeStyle = style.lineColor;
+ this.ctx.lineWidth = style.lineWidth;
+ this.ctx.beginPath();
+ this.ctx.moveTo(proximalPoint.x, proximalPoint.y);
+ this.ctx.lineTo(distalPoint.x, distalPoint.y);
+ this.ctx.stroke();
+ }
+ }
+
+ /**
+ * Generic method to draw a polygon (like ArUco markers)
+ */
+ protected drawPolygon(
+ corners: [number, number][],
+ style: DrawStyle,
+ fillPoints: boolean = true,
+ label?: string
+ ): void {
+ if (corners.length < 3) return;
+
+ // Draw outline
+ this.ctx.strokeStyle = style.lineColor;
+ this.ctx.lineWidth = style.lineWidth;
+ this.ctx.beginPath();
+ this.ctx.moveTo(corners[0][0], corners[0][1]);
+ for (let i = 1; i < corners.length; i++) {
+ this.ctx.lineTo(corners[i][0], corners[i][1]);
+ }
+ this.ctx.closePath();
+ this.ctx.stroke();
+
+ // Draw corner points if requested
+ if (fillPoints) {
+ this.ctx.fillStyle = style.pointColor;
+ for (const corner of corners) {
+ this.ctx.beginPath();
+ this.ctx.arc(corner[0], corner[1], style.pointRadius, 0, Math.PI * 2);
+ this.ctx.fill();
+ }
+ }
+
+ // Draw label at center if provided
+ if (label) {
+ const centerX = corners.reduce((sum, c) => sum + c[0], 0) / corners.length;
+ const centerY = corners.reduce((sum, c) => sum + c[1], 0) / corners.length;
+
+ this.drawText(
+ label,
+ centerX,
+ centerY,
+ style.labelFontSize * 1.2,
+ this.TEXT_COLOR,
+ style.pointColor,
+ 3,
+ 'bold'
+ );
+ }
+ }
+
+ /**
+ * Draw text with stroke (outline)
+ */
+ protected drawText(
+ text: string,
+ x: number,
+ y: number,
+ fontSize: number,
+ fillColor: string,
+ strokeColor: string,
+ strokeWidth: number,
+ fontWeight: string = 'normal'
+ ): void {
+ this.ctx.font = `${fontWeight} ${fontSize}px Arial, sans-serif`;
+ this.ctx.textBaseline = 'middle';
+
+ // Draw stroke (outline)
+ this.ctx.strokeStyle = strokeColor;
+ this.ctx.lineWidth = strokeWidth + 2;
+ this.ctx.strokeText(text, x, y);
+
+ // Draw fill
+ this.ctx.fillStyle = fillColor;
+ this.ctx.fillText(text, x, y);
+ }
+
+ /**
+ * Draw info overlay with metadata
+ */
+ protected drawInfoOverlay(
+ metadata: ObservationMetadata,
+ additionalInfo?: string[]
+ ): void {
+ const lines: string[] = [
+ `Frame: ${metadata.frame_number}`,
+ ...(additionalInfo || [])
+ ];
+
+ // Calculate background size
+ const lineHeight = 25;
+ const bgHeight = lines.length * lineHeight + 10;
+ const bgWidth = 500;
+
+ // Draw background
+ this.ctx.fillStyle = this.INFO_BG;
+ this.ctx.fillRect(5, 5, bgWidth, bgHeight);
+
+ // Draw lines
+ let y = 25;
+ for (const line of lines) {
+ this.drawText(line, 10, y, 14, this.TEXT_COLOR, this.TEXT_STROKE, 2);
+ y += lineHeight;
+ }
+ }
+
+ /**
+ * Check if a point is valid (not NaN or undefined)
+ */
+ protected isValidPoint(point: Point2D): boolean {
+ return (
+ point &&
+ typeof point.x === 'number' &&
+ typeof point.y === 'number' &&
+ !isNaN(point.x) &&
+ !isNaN(point.y) &&
+ isFinite(point.x) &&
+ isFinite(point.y)
+ );
+ }
+
+ /**
+ * Create final bitmap from canvas
+ */
+ protected async createBitmap(sourceBitmap: ImageBitmap): Promise<ImageBitmap> {
+ const compositeBitmap = await createImageBitmap(this.offscreenCanvas);
+ sourceBitmap.close();
+ return compositeBitmap;
+ }
+
+ public destroy(): void {
+ // Cleanup if needed
+ }
+}
\ No newline at end of file
diff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/index.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/index.ts
new file mode 100644
index 00000000..8a7dccfd
--- /dev/null
+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/index.ts
@@ -0,0 +1,20 @@
+// image-overlay/index.ts - Export all overlay components
+export * from './image-overlay-system';
+export * from './charuco-types';
+export * from './charuco-overlay-renderer';
+export * from './overlay-types';
+export * from './overlay-renderer-factory';
+
+// Explicitly re-export to avoid duplicate symbol conflicts
+// (mediapipe-overlay-renderer re-declares types from mediapipe-types)
+export {
+ MediapipePointSchema,
+ MediapipeMetadataSchema,
+ MediapipeOverlaySchema,
+ MediapipeOverlayDataMessageSchema,
+ type MediapipePoint,
+ type MediapipeMetadata,
+ type MediapipeObservation,
+ type MediapipeOverlayDataMessage
+} from './mediapipe-types';
+export { MediapipeOverlayRenderer } from './mediapipe-overlay-renderer';
\ No newline at end of file
diff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/mediapipe-overlay-renderer.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/mediapipe-overlay-renderer.ts
new file mode 100644
index 00000000..59d6ddbc
--- /dev/null
+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/mediapipe-overlay-renderer.ts
@@ -0,0 +1,471 @@
+// mediapipe-overlay-renderer.ts
+
+import {
+ BaseOverlayRenderer,
+ DrawStyle,
+ Point2D
+} from "@/services/server/server-helpers/image-overlay/image-overlay-system";
+
+export interface MediapipePoint {
+ name: string;
+ x: number;
+ y: number;
+ z?: number;
+ visibility?: number;
+}
+
+export interface MediapipeObservation {
+ message_type: 'mediapipe_overlay';
+ camera_id: string;
+ frame_number: number;
+ body_points?: MediapipePoint[];
+ right_hand_points?: MediapipePoint[];
+ left_hand_points?: MediapipePoint[];
+ face_points?: MediapipePoint[];
+ metadata: {
+ image_width: number;
+ image_height: number;
+ n_body_detected: number;
+ n_right_hand_detected: number;
+ n_left_hand_detected: number;
+ n_face_detected: number;
+ };
+}
+
+export class MediapipeOverlayRenderer extends BaseOverlayRenderer {
+ // Body styles split by side
+ private readonly bodyStyleCenter: DrawStyle = {
+ pointColor: '#00AA00',
+ pointStroke: '#008800',
+ pointRadius: 4,
+ lineColor: 'rgba(20, 255, 20, 0.6)',
+ lineWidth: 2,
+ labelColor: '#00AA00',
+ labelStroke: this.TEXT_STROKE,
+ labelFontSize: 8,
+ showLabels: false,
+ };
+
+ private readonly bodyStyleRight: DrawStyle = {
+ pointColor: '#FF4444',
+ pointStroke: '#AA0000',
+ pointRadius: 4,
+ lineColor: 'rgba(255, 68, 68, 0.6)',
+ lineWidth: 2,
+ labelColor: '#FF4444',
+ labelStroke: this.TEXT_STROKE,
+ labelFontSize: 8,
+ showLabels: false,
+ };
+
+ private readonly bodyStyleLeft: DrawStyle = {
+ pointColor: '#4488FF',
+ pointStroke: '#0044AA',
+ pointRadius: 4,
+ lineColor: 'rgba(68, 136, 255, 0.6)',
+ lineWidth: 2,
+ labelColor: '#4488FF',
+ labelStroke: this.TEXT_STROKE,
+ labelFontSize: 8,
+ showLabels: false,
+ };
+
+ private readonly rightHandStyle: DrawStyle = {
+ pointColor: '#FF6400',
+ pointStroke: '#AA4400',
+ pointRadius: 3,
+ lineColor: '#FF6400',
+ lineWidth: 1.5,
+ labelColor: '#FF6400',
+ labelStroke: this.TEXT_STROKE,
+ labelFontSize: 8,
+ showLabels: false,
+ };
+
+ private readonly leftHandStyle: DrawStyle = {
+ pointColor: '#00AAFF',
+ pointStroke: '#0066AA',
+ pointRadius: 3,
+ lineColor: '#00AAFF',
+ lineWidth: 1.5,
+ labelColor: '#00AAFF',
+ labelStroke: this.TEXT_STROKE,
+ labelFontSize: 8,
+ showLabels: false,
+ };
+
+ private readonly faceStyle: DrawStyle = {
+ pointColor: '#FFD700',
+ pointStroke: '#AA9900',
+ pointRadius: 1,
+ lineColor: '#FFD700',
+ lineWidth: 1,
+ labelColor: '#FFD700',
+ labelStroke: this.TEXT_STROKE,
+ labelFontSize: 6,
+ showLabels: false,
+ };
+
+ /**
+ * Composite mediapipe overlay onto frame
+ */
+ public async compositeFrame(
+ sourceBitmap: ImageBitmap,
+ observation: MediapipeObservation | null,
+ ): Promise<ImageBitmap> {
+ this.prepareCanvas(sourceBitmap);
+
+ if (observation) {
+ this.drawMediapipeOverlay(observation);
+ }
+
+ return this.createBitmap(sourceBitmap);
+ }
+
+ private drawMediapipeOverlay(observation: MediapipeObservation): void {
+ this.ctx.save();
+
+ // Draw each aspect in order (face behind, body in middle, hands on top)
+ if (observation.face_points) {
+ this.drawFaceAspect(observation.face_points);
+ }
+
+ if (observation.body_points) {
+ this.drawBodyAspect(observation.body_points);
+ }
+
+ if (observation.right_hand_points) {
+ this.drawHandAspect(observation.right_hand_points, 'right_hand', this.rightHandStyle);
+ }
+
+ if (observation.left_hand_points) {
+ this.drawHandAspect(observation.left_hand_points, 'left_hand', this.leftHandStyle);
+ }
+
+ // Draw info overlay
+ this.drawMediapipeInfo(observation);
+
+ this.ctx.restore();
+ }
+
+ private getBodySide(name: string): 'left' | 'right' | 'center' {
+ if (name.includes('left')) return 'left';
+ if (name.includes('right')) return 'right';
+ return 'center';
+ }
+
+ private getBodyStyleForSide(side: 'left' | 'right' | 'center'): DrawStyle {
+ switch (side) {
+ case 'left': return this.bodyStyleLeft;
+ case 'right': return this.bodyStyleRight;
+ case 'center': return this.bodyStyleCenter;
+ }
+ }
+
+ private drawBodyAspect(points: MediapipePoint[]): void {
+ if (points.length === 0) return;
+
+ // Create point map for segment drawing
+ const pointMap = new Map<string | number, Point2D>();
+ const leftPoints: Point2D[] = [];
+ const rightPoints: Point2D[] = [];
+ const centerPoints: Point2D[] = [];
+
+ for (const point of points) {
+ const point2D: Point2D = {
+ x: point.x,
+ y: point.y,
+ id: point.name,
+ visibility: point.visibility,
+ };
+ pointMap.set(point.name, point2D);
+
+ const side = this.getBodySide(point.name);
+ if (side === 'left') leftPoints.push(point2D);
+ else if (side === 'right') rightPoints.push(point2D);
+ else centerPoints.push(point2D);
+ }
+
+ // Draw body skeleton connections (behind points)
+ this.drawBodyConnections(pointMap);
+
+ // Draw points on top, color-coded by side
+ this.drawPoints(leftPoints, this.bodyStyleLeft);
+ this.drawPoints(rightPoints, this.bodyStyleRight);
+ this.drawPoints(centerPoints, this.bodyStyleCenter);
+
+ // Highlight key points with labels
+ this.drawKeyBodyPoints(pointMap);
+ }
+
+ private drawKeyBodyPoints(pointMap: Map<string | number, Point2D>): void {
+ // Label only important landmarks for clarity
+ const keyPoints = ['body.nose', 'body.left_shoulder', 'body.right_shoulder', 'body.left_hip', 'body.right_hip'];
+
+ for (const key of keyPoints) {
+ const point = pointMap.get(key);
+ if (point && this.isValidPoint(point)) {
+ const style = this.getBodyStyleForSide(this.getBodySide(key));
+ this.drawText(
+ key.replace('body.', '').replace('_', ' '),
+ point.x + 5,
+ point.y - 5,
+ 8,
+ style.labelColor,
+ style.labelStroke,
+ 1
+ );
+ }
+ }
+ }
+
+ private drawBodyConnections(
+ pointMap: Map<string | number, Point2D>,
+ ): void {
+ // Connections grouped by side for color coding
+ const centerConnections = [
+ ['body.nose', 'body.left_eye_inner'],
+ ['body.nose', 'body.right_eye_inner'],
+ ['body.mouth_left', 'body.mouth_right'],
+ ['body.left_shoulder', 'body.right_shoulder'],
+ ['body.left_hip', 'body.right_hip'],
+ ];
+
+ const leftConnections = [
+ ['body.left_eye_inner', 'body.left_eye'],
+ ['body.left_eye', 'body.left_eye_outer'],
+ ['body.left_eye_outer', 'body.left_ear'],
+ ['body.left_shoulder', 'body.left_hip'],
+ ['body.left_shoulder', 'body.left_elbow'],
+ ['body.left_elbow', 'body.left_wrist'],
+ ['body.left_wrist', 'body.left_pinky'],
+ ['body.left_wrist', 'body.left_index'],
+ ['body.left_wrist', 'body.left_thumb'],
+ ['body.left_index', 'body.left_pinky'],
+ ['body.left_hip', 'body.left_knee'],
+ ['body.left_knee', 'body.left_ankle'],
+ ['body.left_ankle', 'body.left_heel'],
+ ['body.left_heel', 'body.left_foot_index'],
+ ['body.left_ankle', 'body.left_foot_index'],
+ ];
+
+ const rightConnections = [
+ ['body.right_eye_inner', 'body.right_eye'],
+ ['body.right_eye', 'body.right_eye_outer'],
+ ['body.right_eye_outer', 'body.right_ear'],
+ ['body.right_shoulder', 'body.right_hip'],
+ ['body.right_shoulder', 'body.right_elbow'],
+ ['body.right_elbow', 'body.right_wrist'],
+ ['body.right_wrist', 'body.right_pinky'],
+ ['body.right_wrist', 'body.right_index'],
+ ['body.right_wrist', 'body.right_thumb'],
+ ['body.right_index', 'body.right_pinky'],
+ ['body.right_hip', 'body.right_knee'],
+ ['body.right_knee', 'body.right_ankle'],
+ ['body.right_ankle', 'body.right_heel'],
+ ['body.right_heel', 'body.right_foot_index'],
+ ['body.right_ankle', 'body.right_foot_index'],
+ ];
+
+ const groups: [string[][], DrawStyle][] = [
+ [centerConnections, this.bodyStyleCenter],
+ [leftConnections, this.bodyStyleLeft],
+ [rightConnections, this.bodyStyleRight],
+ ];
+
+ for (const [connections, style] of groups) {
+ for (const [start, end] of connections) {
+ const startPoint = pointMap.get(start);
+ const endPoint = pointMap.get(end);
+
+ if (startPoint && endPoint && this.isValidPoint(startPoint) && this.isValidPoint(endPoint)) {
+ this.ctx.strokeStyle = style.lineColor;
+ this.ctx.lineWidth = style.lineWidth;
+ this.ctx.beginPath();
+ this.ctx.moveTo(startPoint.x, startPoint.y);
+ this.ctx.lineTo(endPoint.x, endPoint.y);
+ this.ctx.stroke();
+ }
+ }
+ }
+ }
+
+ private drawHandAspect(
+ points: MediapipePoint[],
+ aspectName: string,
+ style: DrawStyle
+ ): void {
+ if (points.length === 0) return;
+
+ const pointMap = new Map<string | number, Point2D>();
+ const point2DArray: Point2D[] = [];
+
+ for (const point of points) {
+ const point2D: Point2D = {
+ x: point.x,
+ y: point.y,
+ id: point.name,
+ visibility: point.visibility,
+ };
+ pointMap.set(point.name, point2D);
+ point2DArray.push(point2D);
+ }
+
+ // Draw hand skeleton connections
+ this.drawHandConnections(pointMap, style);
+
+ // Draw points
+ this.drawPoints(point2DArray, style);
+ }
+
+ private drawHandConnections(
+ pointMap: Map<string | number, Point2D>,
+ style: DrawStyle
+ ): void {
+ // Define hand connections (MediaPipe hand landmark connections)
+ const handConnections = [
+ // Thumb
+ ['wrist', 'thumb_cmc'],
+ ['thumb_cmc', 'thumb_mcp'],
+ ['thumb_mcp', 'thumb_ip'],
+ ['thumb_ip', 'thumb_tip'],
+ // Index finger
+ ['wrist', 'index_finger_mcp'],
+ ['index_finger_mcp', 'index_finger_pip'],
+ ['index_finger_pip', 'index_finger_dip'],
+ ['index_finger_dip', 'index_finger_tip'],
+ // Middle finger
+ ['wrist', 'middle_finger_mcp'],
+ ['middle_finger_mcp', 'middle_finger_pip'],
+ ['middle_finger_pip', 'middle_finger_dip'],
+ ['middle_finger_dip', 'middle_finger_tip'],
+ // Ring finger
+ ['wrist', 'ring_finger_mcp'],
+ ['ring_finger_mcp', 'ring_finger_pip'],
+ ['ring_finger_pip', 'ring_finger_dip'],
+ ['ring_finger_dip', 'ring_finger_tip'],
+ // Pinky
+ ['wrist', 'pinky_mcp'],
+ ['pinky_mcp', 'pinky_pip'],
+ ['pinky_pip', 'pinky_dip'],
+ ['pinky_dip', 'pinky_tip'],
+ // Palm connections
+ ['index_finger_mcp', 'middle_finger_mcp'],
+ ['middle_finger_mcp', 'ring_finger_mcp'],
+ ['ring_finger_mcp', 'pinky_mcp'],
+ ];
+
+ for (const [start, end] of handConnections) {
+ const startPoint = pointMap.get(start);
+ const endPoint = pointMap.get(end);
+
+ if (startPoint && endPoint && this.isValidPoint(startPoint) && this.isValidPoint(endPoint)) {
+ this.ctx.strokeStyle = style.lineColor;
+ this.ctx.lineWidth = style.lineWidth;
+ this.ctx.beginPath();
+ this.ctx.moveTo(startPoint.x, startPoint.y);
+ this.ctx.lineTo(endPoint.x, endPoint.y);
+ this.ctx.stroke();
+ }
+ }
+ }
+
+ private drawFaceAspect(points: MediapipePoint[]): void {
+ if (points.length === 0) return;
+
+ // For face, we just draw the points as a mesh without connections
+ // (too many points to draw connections clearly)
+ const point2DArray: Point2D[] = points.map(point => ({
+ x: point.x,
+ y: point.y,
+ id: point.name,
+ visibility: point.visibility,
+ }));
+
+ this.drawPoints(point2DArray, this.faceStyle);
+
+ // Draw face contour highlights for key features
+ this.drawFaceContour(points);
+ }
+
+ private drawFaceContour(points: MediapipePoint[]): void {
+ // Draw simplified face contour for better visibility
+ // This would use the face contour indices from the model info
+ // For now, just highlight key face points
+ const keyFacePoints = points.filter(p =>
+ p.name.includes('eye') ||
+ p.name.includes('mouth') ||
+ p.name.includes('nose')
+ );
+
+ const highlightStyle = {
+ ...this.faceStyle,
+ pointRadius: 2,
+ pointColor: '#FFFF00',
+ };
+
+ this.drawPoints(
+ keyFacePoints.map(p => ({
+ x: p.x,
+ y: p.y,
+ id: p.name,
+ visibility: p.visibility,
+ })),
+ highlightStyle
+ );
+ }
+
+ private drawMediapipeInfo(observation: MediapipeObservation): void {
+ const { metadata, frame_number } = observation;
+
+ // Build detection status
+ const detections: string[] = [];
+ if (metadata.n_body_detected > 0) detections.push('Body');
+ if (metadata.n_right_hand_detected > 0) detections.push('R-Hand');
+ if (metadata.n_left_hand_detected > 0) detections.push('L-Hand');
+ if (metadata.n_face_detected > 0) detections.push('Face');
+
+ const statusText = detections.length > 0
+ ? `βœ” Detected: ${detections.join(', ')}`
+ : '⚠ No detection';
+
+ const statusColor = detections.length > 0 ? '#00FF00' : '#FF4444';
+
+ // Draw info background
+ this.ctx.fillStyle = this.INFO_BG;
+ this.ctx.fillRect(5, 5, 500, 85);
+
+ // Draw frame info
+ this.drawText(
+ `Frame: ${frame_number}`,
+ 10,
+ 25,
+ 14,
+ this.TEXT_COLOR,
+ this.TEXT_STROKE,
+ 2
+ );
+
+ // Draw detection counts
+ this.drawText(
+ `Body: ${metadata.n_body_detected} | Hands: ${metadata.n_right_hand_detected + metadata.n_left_hand_detected} | Face: ${metadata.n_face_detected > 0 ? 'βœ”' : 'βœ—'}`,
+ 10,
+ 50,
+ 14,
+ this.TEXT_COLOR,
+ this.TEXT_STROKE,
+ 2
+ );
+
+ // Draw status
+ this.drawText(
+ statusText,
+ 10,
+ 75,
+ 16,
+ statusColor,
+ this.TEXT_STROKE,
+ 2
+ );
+ }
+}
\ No newline at end of file
diff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/mediapipe-types.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/mediapipe-types.ts
new file mode 100644
index 00000000..f103f700
--- /dev/null
+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/mediapipe-types.ts
@@ -0,0 +1,41 @@
+import z from "zod";
+
+// Individual point schema
+export const MediapipePointSchema = z.object({
+ name: z.string(),
+ x: z.number(),
+ y: z.number(),
+ z: z.number(),
+ visibility: z.number(),
+});
+
+// Metadata schema
+export const MediapipeMetadataSchema = z.object({
+ n_body_detected: z.number(),
+ n_right_hand_detected: z.number(),
+ n_left_hand_detected: z.number(),
+ n_face_detected: z.number(),
+ image_width: z.number(),
+ image_height: z.number(),
+});
+
+// Single camera observation schema
+export const MediapipeOverlaySchema = z.object({
+ message_type: z.literal("mediapipe_overlay"),
+ camera_id: z.string(),
+ frame_number: z.number(),
+ body_points: z.array(MediapipePointSchema),
+ right_hand_points: z.array(MediapipePointSchema),
+ left_hand_points: z.array(MediapipePointSchema),
+ face_points: z.array(MediapipePointSchema),
+ metadata: MediapipeMetadataSchema,
+});
+
+// Multi-camera message schema (matches CharucoOverlayDataMessage structure)
+export const MediapipeOverlayDataMessageSchema = z.record(z.string(), MediapipeOverlaySchema);
+
+// Type exports
+export type MediapipePoint = z.infer<typeof MediapipePointSchema>;
+export type MediapipeMetadata = z.infer<typeof MediapipeMetadataSchema>;
+export type MediapipeObservation = z.infer<typeof MediapipeOverlaySchema>;
+export type MediapipeOverlayDataMessage = z.infer<typeof MediapipeOverlayDataMessageSchema>;
\ No newline at end of file
diff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/overlay-renderer-factory.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/overlay-renderer-factory.ts
new file mode 100644
index 00000000..060463b3
--- /dev/null
+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/overlay-renderer-factory.ts
@@ -0,0 +1,145 @@
+// overlay-renderer-factory.ts
+import { CharucoOverlayRenderer } from './charuco-overlay-renderer';
+import { MediapipeOverlayRenderer, MediapipeObservation } from './mediapipe-overlay-renderer';
+import {BaseOverlayRenderer, ModelInfo} from "@/services/server/server-helpers/image-overlay/image-overlay-system";
+import {CharucoObservation} from "@/services/server/server-helpers/image-overlay/charuco-types";
+
+export type ObservationType = 'charuco_overlay' | 'mediapipe_overlay' | 'rtmpose_overlay';
+
+
+/**
+ * Factory for creating appropriate overlay renderers based on observation type
+ */
+export class OverlayRendererFactory {
+ private static rendererCache: Map<ObservationType, BaseOverlayRenderer> = new Map();
+ private static modelInfoCache: Map<string, ModelInfo> = new Map();
+
+ /**
+ * Get or create a renderer for the specified observation type
+ */
+ public static getRenderer(type: ObservationType): BaseOverlayRenderer {
+ // Check cache first
+ let renderer = this.rendererCache.get(type);
+ if (renderer) {
+ return renderer;
+ }
+
+ // Create new renderer based on type
+ switch (type) {
+ case 'charuco_overlay':
+ renderer = new CharucoOverlayRenderer();
+ break;
+ case 'mediapipe_overlay':
+ renderer = new MediapipeOverlayRenderer();
+ break;
+ case 'rtmpose_overlay':
+ // Future: implement RTMPoseOverlayRenderer
+ throw new Error('RTMPose overlay not yet implemented');
+ default:
+ throw new Error(`Unknown observation type: ${type}`);
+ }
+
+ this.rendererCache.set(type, renderer);
+ return renderer;
+ }
+
+ /**
+ * Set model info for a specific tracker type
+ */
+ public static setModelInfo(trackerName: string, modelInfo: ModelInfo): void {
+ this.modelInfoCache.set(trackerName, modelInfo);
+
+ // Update all renderers that might use this model info
+ for (const [type, renderer] of this.rendererCache) {
+ if (this.shouldUseModelInfo(type, trackerName)) {
+ renderer.setModelInfo(modelInfo);
+ }
+ }
+ }
+
+ /**
+ * Clear all cached renderers
+ */
+ public static clearCache(): void {
+ for (const renderer of this.rendererCache.values()) {
+ renderer.destroy();
+ }
+ this.rendererCache.clear();
+ this.modelInfoCache.clear();
+ }
+
+ /**
+ * Determine if a renderer type should use a specific model info
+ */
+ private static shouldUseModelInfo(
+ rendererType: ObservationType,
+ trackerName: string
+ ): boolean {
+ const mapping: Record<ObservationType, string[]> = {
+ 'charuco_overlay': ['CharucoTracker'],
+ 'mediapipe_overlay': ['MediapipeHolisticTracker', 'MediapipeTracker'],
+ 'rtmpose_overlay': ['RTMPoseTracker'],
+ };
+
+ return mapping[rendererType]?.includes(trackerName) || false;
+ }
+}
+
+/**
+ * Manager for handling overlay rendering across multiple cameras.
+ *
+ * Composites both charuco and mediapipe overlays sequentially onto each
+ * frame: source β†’ charuco β†’ mediapipe. Each renderer draws on top of
+ * the previous result rather than clearing and redrawing from scratch.
+ */
+export class OverlayManager {
+ private charucoRenderer: CharucoOverlayRenderer = new CharucoOverlayRenderer();
+ private mediapipeRenderer: MediapipeOverlayRenderer = new MediapipeOverlayRenderer();
+
+ /**
+ * Composite both overlay types onto a single frame, chained sequentially.
+ * The charuco overlay is drawn first, then mediapipe on top of the result.
+ */
+ public async processFrame(
+ cameraId: string,
+ sourceBitmap: ImageBitmap,
+ charucoObservation: CharucoObservation | null,
+ mediapipeObservation: MediapipeObservation | null,
+ ): Promise<ImageBitmap> {
+ let currentBitmap = sourceBitmap;
+
+ if (charucoObservation) {
+ currentBitmap = await this.charucoRenderer.compositeFrame(
+ currentBitmap,
+ charucoObservation,
+ );
+ }
+
+ if (mediapipeObservation) {
+ currentBitmap = await this.mediapipeRenderer.compositeFrame(
+ currentBitmap,
+ mediapipeObservation,
+ );
+ }
+
+ return currentBitmap;
+ }
+
+ /**
+ * Set model info on both renderers
+ */
+ public setModelInfo(modelInfo: ModelInfo): void {
+ this.charucoRenderer.setModelInfo(modelInfo);
+ this.mediapipeRenderer.setModelInfo(modelInfo);
+ }
+
+ /**
+ * Clear all renderers
+ */
+ public clearAll(): void {
+ this.charucoRenderer.destroy();
+ this.mediapipeRenderer.destroy();
+ this.charucoRenderer = new CharucoOverlayRenderer();
+ this.mediapipeRenderer = new MediapipeOverlayRenderer();
+ }
+}
\ No newline at end of file
diff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/overlay-types.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/overlay-types.ts
new file mode 100644
index 00000000..7bf5c2e5
--- /dev/null
+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/overlay-types.ts
@@ -0,0 +1,17 @@
+import {CharucoObservation, CharucoOverlayDataMessage} from "@/services/server/server-helpers/image-overlay/charuco-types";
+import {MediapipeObservation, MediapipeOverlayDataMessage} from "@/services/server/server-helpers/image-overlay/mediapipe-types";
+
+// Union type for all observation data messages
+export type ObservationDataMessage = CharucoOverlayDataMessage | MediapipeOverlayDataMessage;
+
+// Union type for individual observations
+export type Observation = CharucoObservation | MediapipeObservation;
+
+// Type guard functions
+export function isCharucoObservation(obs: Observation): obs is CharucoObservation {
+ return obs.message_type === 'charuco_overlay';
+}
+
+export function isMediapipeObservation(obs: Observation): obs is MediapipeObservation {
+ return obs.message_type === 'mediapipe_overlay';
+}
\ No newline at end of file
diff --git a/freemocap-ui/src/services/server/server-helpers/websocket-message-types.ts b/freemocap-ui/src/services/server/server-helpers/websocket-message-types.ts
new file mode 100644
index 00000000..59130c97
--- /dev/null
+++ b/freemocap-ui/src/services/server/server-helpers/websocket-message-types.ts
@@ -0,0 +1,101 @@
+import {
+ CharucoOverlayDataMessage,
+ CharucoOverlayDataMessageSchema,
+} from "@/services/server/server-helpers/image-overlay/charuco-types";
+import {
+ MediapipeOverlayDataMessage,
+ MediapipeOverlayDataMessageSchema,
+} from "@/services/server/server-helpers/image-overlay/mediapipe-types";
+import {ModelInfo} from "@/services/server/server-helpers/image-overlay/image-overlay-system";
+import {OverlayRendererFactory} from "@/services/server/server-helpers/image-overlay/overlay-renderer-factory";
+
+export interface FramerateUpdateMessage {
+ message_type: 'framerate_update';
+ camera_group_id: string;
+ backend_framerate: {
+ mean: number;
+ std: number;
+ min: number;
+ max: number;
+ median: number;
+ recent: number[];
+ };
+ frontend_framerate: {
+ mean: number;
+ std: number;
+ min: number;
+ max: number;
+ median: number;
+ recent: number[];
+ };
+}
+
+export interface LogRecord {
+ message_type: 'log_record';
+ levelname: string;
+ message: string;
+ timestamp: string;
+}
+
+export interface SettingsStateMessage {
+ message_type: 'settings/state';
+ settings: Record<string, unknown>;
+ version: number;
+}
+
+export function arraysEqual(a: string[], b: string[]): boolean {
+ if (a.length !== b.length) return false;
+ const sortedA = [...a].sort();
+ const sortedB = [...b].sort();
+ return sortedA.every((val, idx) => val === sortedB[idx]);
+}
+
+export function isLogRecord(data: unknown): data is LogRecord {
+ return (
+ typeof data === 'object' &&
+ data !== null &&
+ (data as Record<string, unknown>).message_type === 'log_record' &&
+ typeof (data as Record<string, unknown>).levelname === 'string' &&
+ typeof (data as Record<string, unknown>).message === 'string'
+ );
+}
+
+export function isSettingsStateMessage(data: unknown): data is SettingsStateMessage {
+ return (
+ typeof data === 'object' &&
+ data !== null &&
+ (data as Record<string, unknown>).message_type === 'settings/state' &&
+ typeof (data as Record<string, unknown>).settings === 'object' &&
+ (data as Record<string, unknown>).settings !== null &&
+ typeof (data as Record<string, unknown>).version === 'number'
+ );
+}
+
+export function isFramerateUpdate(data: unknown): data is FramerateUpdateMessage {
+ const d = data as Record<string, unknown>;
+ return (
+ typeof data === 'object' &&
+ data !== null &&
+ d.message_type === 'framerate_update' &&
+ typeof d.camera_group_id === 'string' &&
+ typeof d.backend_framerate === 'object' &&
+ d.backend_framerate !== null &&
+ typeof d.frontend_framerate === 'object' &&
+ d.frontend_framerate !== null
+ );
+}
+
+export function isCharucoOverlayDataMessage(data: unknown): data is CharucoOverlayDataMessage {
+ const result = CharucoOverlayDataMessageSchema.safeParse(data);
+ return result.success;
+}
+
+export function isMediapipeOverlayDataMessage(data: unknown): data is MediapipeOverlayDataMessage {
+ const result = MediapipeOverlayDataMessageSchema.safeParse(data);
+ return result.success;
+}
+
+export function handleModelInfoUpdate(modelInfo: ModelInfo): void {
+ console.log(`Received model info for tracker: ${modelInfo.tracker_name}`);
+ OverlayRendererFactory.setModelInfo(modelInfo.tracker_name, modelInfo);
+}
\ No newline at end of file
diff --git a/freemocap/api/websocket/websocket_server.py b/freemocap/api/websocket/websocket_server.py
index 5e1169a6..5f8b238a 100644
--- a/freemocap/api/websocket/websocket_server.py
+++ b/freemocap/api/websocket/websocket_server.py
@@ -189,6 +189,10 @@ class WebsocketServer:
point_name: tracked_point.model_dump()
for point_name, tracked_point in frontend_payload.tracked_points3d.items()
}
+ for point_name, point in points3d_dict.items():
+ point['x'] *= 0.001
+ point['y'] *= 0.001
+ point['z'] *= 0.001
async with self._send_lock:
await self.websocket.send_json({"tracked_points3d": points3d_dict})
@@ -316,4 +320,4 @@ class WebsocketServer:
self._websocket_should_continue = False
raise
finally:
- logger.info("Ending client message handler...")
\ No newline at end of file
+ logger.info("Ending client message handler...")
diff --git a/shared/freemocap-logo/freemocap-favicon.ico b/shared/freemocap-logo/freemocap-favicon.ico
new file mode 100644
index 00000000..ade9aa25
Binary files /dev/null and b/shared/freemocap-logo/freemocap-favicon.ico differ
diff --git a/shared/freemocap-logo/freemocap-logo.ico b/shared/freemocap-logo/freemocap-logo.ico
deleted file mode 100644
index f5271037..00000000
Binary files a/shared/freemocap-logo/freemocap-logo.ico and /dev/null differ