Free Motion Capture for Everyone πβ¨
TypeScript
add overlay view and clean up sidebar stuff
jonmatthisf001843cb4af
diff --git a/freemocap-ui/src/components/pipeline-config/PipelineConfigTree.tsx b/freemocap-ui/src/components/pipeline-config/PipelineConfigTree.tsxindex 7463499e..412f3834 100644--- a/freemocap-ui/src/components/pipeline-config/PipelineConfigTree.tsx+++ b/freemocap-ui/src/components/pipeline-config/PipelineConfigTree.tsx@@ -24,9 +24,6 @@ interface PipelineConfigTreeProps {onFilterToggle: (checked: boolean) => void;rigidBodyEnabled: boolean;onRigidBodyToggle: (checked: boolean) => void;- // Disable all controls (e.g. pipeline not connected)- disabled?: boolean;- disabledReason?: string;}export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({@@ -41,8 +38,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({onFilterToggle,rigidBodyEnabled,onRigidBodyToggle,- disabled = false,- disabledReason,}) => {const theme = useTheme();const [expandedItems, setExpandedItems] = useState<string[]>([]);@@ -79,8 +74,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({label="2D Tracking"checked={tracking2dEnabled}onToggle={handleToggle2dTracking}- disabled={disabled}- disabledReason={disabledReason}isExpanded={isExpanded("2d-tracking")}summaryWhenCollapsed={[charucoEnabled && "Charuco", skeletonEnabled && "Skeleton"]@@ -94,8 +87,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({label="Charuco"checked={charucoEnabled}onToggle={onCharucoToggle}- disabled={disabled}- disabledReason={disabledReason}isExpanded={isExpanded("2d-charuco")}><Box sx={{p: 1, pl: 2}}>@@ -111,8 +102,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({label="Skeleton"checked={skeletonEnabled}onToggle={onSkeletonToggle}- disabled={disabled}- disabledReason={disabledReason}isExpanded={isExpanded("2d-skeleton")}summaryWhenCollapsed={context === "realtime" ? "Realtime preset" : "Posthoc preset"}>@@ -128,8 +117,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({label="3D Reconstruction"checked={reconstruction3dEnabled}onToggle={handleToggle3dReconstruction}- disabled={disabled}- disabledReason={disabledReason}isExpanded={isExpanded("3d-reconstruction")}summaryWhenCollapsed={[triangulateEnabled && "Triangulate", filterEnabled && "Filter", rigidBodyEnabled && "Skeleton"]@@ -143,8 +130,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({label="Triangulate"checked={triangulateEnabled}onToggle={onTriangulateToggle}- disabled={disabled}- disabledReason={disabledReason}isExpanded={isExpanded("3d-triangulate")}><Box sx={{p: 1, pl: 2}}>@@ -160,8 +145,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({label="Filter"checked={filterEnabled}onToggle={onFilterToggle}- disabled={disabled}- disabledReason={disabledReason}isExpanded={isExpanded("3d-filter")}summaryWhenCollapsed="One Euro, FABRIK">@@ -176,8 +159,6 @@ export const PipelineConfigTree: React.FC<PipelineConfigTreeProps> = ({label="Skeleton"checked={rigidBodyEnabled}onToggle={onRigidBodyToggle}- disabled={disabled}- disabledReason={disabledReason}isExpanded={isExpanded("3d-skeleton")}><Box sx={{p: 1, pl: 2}}>diff --git a/freemocap-ui/src/components/processing-pipeline-panel/ProcessingPipelinePanel.tsx b/freemocap-ui/src/components/processing-pipeline-panel/ProcessingPipelinePanel.tsxindex 0bc5e19e..f28fe7b2 100644--- a/freemocap-ui/src/components/processing-pipeline-panel/ProcessingPipelinePanel.tsx+++ b/freemocap-ui/src/components/processing-pipeline-panel/ProcessingPipelinePanel.tsx@@ -14,8 +14,7 @@ export const ProcessingPipelinePanel: React.FC = () => {const backendPipeline = useAppSelector(selectBackendPipeline);const pipelineConfig = backendPipeline?.config;- const isPipelineConnected = backendPipeline?.is_connected ?? false;- const canToggle = isConnected && isPipelineConnected;+// 3D Reconstruction toggles β local state until backend support is addedconst [triangulateEnabled, setTriangulateEnabled] = useState(false);@@ -38,11 +37,7 @@ export const ProcessingPipelinePanel: React.FC = () => {defaultExpanded={false}><Box sx={{p: 2}}>- {!isPipelineConnected && (- <Typography variant="caption" color="text.secondary" sx={{fontStyle: "italic", mb: 1, display: "block"}}>- Connect a pipeline to enable detection toggles.- </Typography>- )}+<PipelineConfigTreecontext="realtime"charucoEnabled={pipelineConfig?.calibration_detection_enabled ?? true}@@ -55,7 +50,6 @@ export const ProcessingPipelinePanel: React.FC = () => {onFilterToggle={setFilterEnabled}rigidBodyEnabled={rigidBodyEnabled}onRigidBodyToggle={setRigidBodyEnabled}- disabled={!canToggle}disabledReason="Connect a pipeline to enable detection toggles"/></Box>diff --git a/freemocap-ui/src/services/server/ServerContextProvider.tsx b/freemocap-ui/src/services/server/ServerContextProvider.tsxindex bc4ea8e7..f1e95842 100644--- a/freemocap-ui/src/services/server/ServerContextProvider.tsx+++ b/freemocap-ui/src/services/server/ServerContextProvider.tsx@@ -8,6 +8,13 @@ import { serverUrls } from "@/services";import {DetailedFramerate, FramerateStore} from "@/services/server/server-helpers/framerate-store";import {LogStore, LogRecord} from "@/services/server/server-helpers/log-store";import { Point3d, RigidBodyPose } from "@/components/viewport3d/viewport3d-types";+import { OverlayManager } from "@/services/server/server-helpers/image-overlay/overlay-renderer-factory";+import { CharucoObservation } from "@/services/server/server-helpers/image-overlay/charuco-types";+import { MediapipeObservation } from "@/services/server/server-helpers/image-overlay/mediapipe-types";+import {+ isCharucoOverlayDataMessage,+ isMediapipeOverlayDataMessage+} from "@/services/server/server-helpers/websocket-message-types";interface ServerContextValue {isConnected: boolean;@@ -81,6 +88,11 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ childconst canvasManagerRef = useRef<CanvasManager | null>(null);const framerateStoreRef = useRef<FramerateStore>(new FramerateStore());const logStoreRef = useRef<LogStore>(new LogStore());+ const overlayManagerRef = useRef<OverlayManager | null>(null);++ // Overlay data refs - NO REACT STATE to avoid re-renders!+ const latestCharucoRef = useRef<Map<string, CharucoObservation>>(new Map());+ const latestMediapipeRef = useRef<Map<string, MediapipeObservation>>(new Map());// Latest server-side (backend) FPS stored in a ref for non-reactive accessconst serverFpsRef = useRef<number | null>(null);@@ -116,6 +128,7 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ child});frameProcessorRef.current = new FrameProcessor();canvasManagerRef.current = new CanvasManager();+ overlayManagerRef.current = new OverlayManager();return () => {if (wsConnectionRef.current) {@@ -127,6 +140,7 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ childif (frameProcessorRef.current) {frameProcessorRef.current.reset();}+ overlayManagerRef.current?.clearAll();};}, []);@@ -149,20 +163,22 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ childframerateStoreRef.current.clear();trackedPointsRef.current = new Map();rigidBodiesRef.current = new Map();+ latestCharucoRef.current.clear();+ latestMediapipeRef.current.clear();+ overlayManagerRef.current?.clearAll();setConnectedCameraIds([]);}};// Process a decoded frame result: update camera list, dispatch to workers, send ack.- const dispatchFrames = (+ const dispatchFrames = async (result: Awaited<ReturnType<FrameProcessor['processFramePayload']>>- ): void => {+ ): Promise<void> => {if (!result) return;const { frames, cameraIds, frameNumbers } = result;// Only allocate a new sorted array if the camera set actually changed.- // Compare against the cached ref to avoid Array.from().sort() on every frame.const lastIds = lastCameraIdsRef.current;let cameraListChanged = lastIds.length !== cameraIds.size;if (!cameraListChanged) {@@ -183,6 +199,8 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ childconst removedCameras = prevIds.filter(id => !cameraIds.has(id));for (const cameraId of removedCameras) {canvasManagerRef.current?.terminateWorker(cameraId);+ latestCharucoRef.current.delete(cameraId);+ latestMediapipeRef.current.delete(cameraId);}return newIds;}@@ -190,10 +208,27 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ child});}+ // Composite overlays onto frames BEFORE sending to canvas workers+ const overlayManager = overlayManagerRef.current!;for (const frameData of frames) {+ const charucoObs = latestCharucoRef.current.get(frameData.cameraId) ?? null;+ const mediapipeObs = latestMediapipeRef.current.get(frameData.cameraId) ?? null;++ let compositeBitmap: ImageBitmap;+ if (charucoObs || mediapipeObs) {+ compositeBitmap = await overlayManager.processFrame(+ frameData.cameraId,+ frameData.bitmap,+ charucoObs,+ mediapipeObs,+ );+ } else {+ compositeBitmap = frameData.bitmap;+ }+canvasManagerRef.current!.sendFrameToWorker(frameData.cameraId,- frameData.bitmap+ compositeBitmap,);}@@ -213,7 +248,7 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ childprocessingFrameRef.current = true;try {const result = await frameProcessorRef.current!.processFramePayload(payload);- dispatchFrames(result);+ await dispatchFrames(result);} catch (error) {console.error('Error processing frame:', error);} finally {@@ -251,6 +286,18 @@ export const ServerContextProvider: React.FC<{ children: ReactNode }> = ({ childframerateStoreRef.current.updateBackend(jsonData.backend_framerate);framerateStoreRef.current.updateFrontend(jsonData.frontend_framerate);}+ // Handle charuco overlay data+ else if (isCharucoOverlayDataMessage(jsonData)) {+ for (const [cameraId, observation] of Object.entries(jsonData)) {+ latestCharucoRef.current.set(cameraId, observation as CharucoObservation);+ }+ }+ // Handle mediapipe overlay data+ else if (isMediapipeOverlayDataMessage(jsonData)) {+ for (const [cameraId, observation] of Object.entries(jsonData)) {+ latestMediapipeRef.current.set(cameraId, observation as MediapipeObservation);+ }+ }// Handle tracked 3D pointselse if (jsonData.tracked_points3d && typeof jsonData.tracked_points3d === 'object') {const pointsMap = new Map<string, Point3d>();@@ -387,4 +434,4 @@ export const useServer = (): ServerContextValue => {const context = useContext(ServerContext);if (!context) throw new Error('useServer must be used within ServerContextProvider');return context;-};+};\ No newline at end of filediff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/charuco-overlay-renderer.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/charuco-overlay-renderer.tsnew file mode 100644index 00000000..a9fd2b27--- /dev/null+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/charuco-overlay-renderer.ts@@ -0,0 +1,184 @@+// charuco-overlay-renderer.ts++import {+ BaseOverlayRenderer,+ DrawStyle,+ Point2D+} from "@/services/server/server-helpers/image-overlay/image-overlay-system";+import {+ ArucoMarker,+ CharucoObservation,+ CharucoPoint+} from "@/services/server/server-helpers/image-overlay/charuco-types";++export class CharucoOverlayRenderer extends BaseOverlayRenderer {+ // Charuco-specific styles+ private readonly charucoStyle: DrawStyle = {+ pointColor: '#00FF00',+ pointStroke: '#009600',+ pointRadius: 5,+ lineColor: '#00FF00',+ lineWidth: 2,+ labelColor: '#00FF00',+ labelStroke: this.TEXT_STROKE,+ labelFontSize: 10,+ showLabels: true,+ };++ private readonly arucoStyle: DrawStyle = {+ pointColor: '#FF6400',+ pointStroke: '#C85000',+ pointRadius: 4,+ lineColor: '#FF6400',+ lineWidth: 3,+ labelColor: this.TEXT_COLOR,+ labelStroke: '#FF6400',+ labelFontSize: 14,+ showLabels: true,+ };++ /**+ * Composite charuco overlay onto frame+ */+ public async compositeFrame(+ sourceBitmap: ImageBitmap,+ observation: CharucoObservation | null,+ ): Promise<ImageBitmap> {+ this.prepareCanvas(sourceBitmap);++ if (observation) {+ this.drawCharucoOverlay(observation);+ }++ return this.createBitmap(sourceBitmap);+ }++ private drawCharucoOverlay(observation: CharucoObservation): void {+ this.ctx.save();++ // Draw ArUco markers first (behind Charuco corners)+ this.drawArucoMarkers(observation.aruco_markers);++ // Draw Charuco corners+ this.drawCharucoCorners(observation.charuco_corners);++ // Draw segments if model info is available+ if (this.modelInfo) {+ this.drawCharucoSegments(observation.charuco_corners);+ }++ // Draw info overlay+ this.drawCharucoInfo(observation);++ this.ctx.restore();+ }++ private drawCharucoCorners(corners: CharucoPoint[]): void {+ if (corners.length === 0) return;++ // Convert CharucoPoint[] to Point2D[]+ const points: Point2D[] = corners.map(corner => ({+ x: corner.x,+ y: corner.y,+ id: corner.id,+ }));++ this.drawPoints(points, this.charucoStyle);+ }++ private drawCharucoSegments(corners: CharucoPoint[]): void {+ if (!this.modelInfo || corners.length === 0) return;++ // Create a map of corner ID to Point2D+ const pointMap = new Map<string | number, Point2D>();+ for (const corner of corners) {+ pointMap.set(corner.id.toString(), {+ x: corner.x,+ y: corner.y,+ id: corner.id,+ });+ }++ // Get segments from model info+ const bodyAspect = this.modelInfo.aspects['body'];+ if (bodyAspect?.segment_connections) {+ this.drawSegments(pointMap, bodyAspect.segment_connections, {+ ...this.charucoStyle,+ lineColor: 'rgba(0, 255, 0, 0.5)', // Semi-transparent green+ lineWidth: 1,+ });+ }+ }++ private drawArucoMarkers(markers: ArucoMarker[]): void {+ if (markers.length === 0) return;++ for (const marker of markers) {+ this.drawPolygon(+ marker.corners,+ this.arucoStyle,+ true,+ marker.id.toString()+ );+ }+ }++ private drawCharucoInfo(observation: CharucoObservation): void {+ const { metadata, frame_number } = observation;++ // Build status info+ let statusText = '';+ let statusColor = '';+ if (metadata.n_charuco_detected === 0 && metadata.n_aruco_detected === 0) {+ statusText = 'β NO BOARD DETECTED';+ statusColor = '#FF4444';+ } else if (metadata.n_charuco_detected < 4) {+ statusText = 'β INSUFFICIENT CORNERS';+ statusColor = '#FFAA00';+ } else {+ statusText = 'β BOARD DETECTED';+ statusColor = '#00FF00';+ }++ const additionalInfo = [+ `Charuco: ${metadata.n_charuco_detected}/${metadata.n_charuco_total} | ` ++ `ArUco: ${metadata.n_aruco_detected}/${metadata.n_aruco_total}` ++ (metadata.has_pose ? ' | POSE β' : ''),+ statusText,+ ];++ // Draw background and info+ this.ctx.fillStyle = this.INFO_BG;+ this.ctx.fillRect(5, 5, 500, 85);++ this.drawText(+ `Frame: ${frame_number}`,+ 10,+ 25,+ 14,+ this.TEXT_COLOR,+ this.TEXT_STROKE,+ 2+ );++ this.drawText(+ additionalInfo[0],+ 10,+ 50,+ 14,+ this.TEXT_COLOR,+ this.TEXT_STROKE,+ 2+ );++ this.drawText(+ additionalInfo[1],+ 10,+ 75,+ 16,+ statusColor,+ this.TEXT_STROKE,+ 2+ );+ }+}\ No newline at end of filediff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/charuco-types.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/charuco-types.tsnew file mode 100644index 00000000..f3edadbe--- /dev/null+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/charuco-types.ts@@ -0,0 +1,43 @@+import {z} from 'zod';++// Zod schemas for CharucoObservation data+export const CharucoPointSchema = z.object({+ id: z.number(),+ x: z.number(),+ y: z.number(),+});++export const ArucoMarkerSchema = z.object({+ id: z.number(),+ corners: z.array(z.tuple([z.number(), z.number()])).length(4),+});++export const CharucoMetadataSchema = z.object({+ n_charuco_detected: z.number(),+ n_charuco_total: z.number(),+ n_aruco_detected: z.number(),+ n_aruco_total: z.number(),+ has_pose: z.boolean(),+ image_width: z.number(),+ image_height: z.number(),+});++// Schema for a single camera's charuco observation+export const CharucoOverlaySchema = z.object({+ message_type: z.literal("charuco_overlay"),+ camera_id: z.string(),+ frame_number: z.number(),+ charuco_corners: z.array(CharucoPointSchema),+ aruco_markers: z.array(ArucoMarkerSchema),+ metadata: CharucoMetadataSchema,+});++// Schema for the websocket message containing multiple camera observations+// Structure: { "camera_1": {...}, "camera_2": {...}, ... }+export const CharucoOverlayDataMessageSchema = z.record(z.string(), CharucoOverlaySchema);++// Type exports+export type CharucoPoint = z.infer<typeof CharucoPointSchema>;+export type ArucoMarker = z.infer<typeof ArucoMarkerSchema>;+export type CharucoObservation = z.infer<typeof CharucoOverlaySchema>;+export type CharucoOverlayDataMessage = z.infer<typeof CharucoOverlayDataMessageSchema>;\ No newline at end of filediff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/image-overlay-system.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/image-overlay-system.tsnew file mode 100644index 00000000..f0fe2b21--- /dev/null+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/image-overlay-system.ts@@ -0,0 +1,303 @@+// base-overlay-renderer.ts+import { z } from 'zod';++// Generic types for any observation+export interface Point2D {+ x: number;+ y: number;+ id?: string | number;+ visibility?: number; // 0-1, for confidence/visibility+}++export interface Segment {+ proximal: string | number;+ distal: string | number;+}++export interface ModelInfo {+ name: string;+ tracker_name: string;+ aspects: Record<string, AspectInfo>;+ order: string[];+}++export interface AspectInfo {+ tracked_points: {+ type: 'list' | 'generated';+ names?: string[];+ convention?: string;+ count?: number;+ };+ segment_connections?: Record<string, Segment>;+ joint_hierarchy?: Record<string, string[]>;+}++export interface DrawStyle {+ pointColor: string;+ pointStroke: string;+ pointRadius: number;+ lineColor: string;+ lineWidth: number;+ labelColor: string;+ labelStroke: string;+ labelFontSize: number;+ showLabels: boolean;+}++export interface ObservationMetadata {+ frame_number: number;+ [key: string]: any; // Allow additional metadata+}++export abstract class BaseOverlayRenderer {+ protected offscreenCanvas: OffscreenCanvas;+ protected ctx: OffscreenCanvasRenderingContext2D;+ protected modelInfo: ModelInfo | null = null;++ // Common style constants+ protected readonly INFO_BG = 'rgba(0, 0, 0, 0.7)';+ protected readonly TEXT_COLOR = '#ccc';+ protected readonly TEXT_STROKE = '#111';++ constructor() {+ this.offscreenCanvas = new OffscreenCanvas(1, 1);+ const ctx = this.offscreenCanvas.getContext('2d', {+ alpha: true,+ desynchronized: true,+ });++ if (!ctx) {+ throw new Error('Failed to get 2D context for overlay renderer');+ }++ this.ctx = ctx;+ }++ /**+ * Set the model info for this renderer+ */+ public setModelInfo(modelInfo: ModelInfo): void {+ this.modelInfo = modelInfo;+ }++ /**+ * Main composite method - to be implemented by specific renderers+ */+ public abstract compositeFrame(+ sourceBitmap: ImageBitmap,+ observation: any | null,+ ): Promise<ImageBitmap>;++ /**+ * Resize canvas if needed and draw source image+ */+ protected prepareCanvas(sourceBitmap: ImageBitmap): void {+ const { width, height } = sourceBitmap;++ if (this.offscreenCanvas.width !== width || this.offscreenCanvas.height !== height) {+ this.offscreenCanvas.width = width;+ this.offscreenCanvas.height = height;+ }++ this.ctx.clearRect(0, 0, width, height);+ this.ctx.drawImage(sourceBitmap, 0, 0);+ }++ /**+ * Generic method to draw points+ */+ protected drawPoints(+ points: Point2D[],+ style: DrawStyle+ ): void {+ for (const point of points) {+ // Skip invalid points+ if (!this.isValidPoint(point)) continue;++ // Optional visibility threshold+ if (point.visibility !== undefined && point.visibility < 0.5) continue;++ // Draw filled circle+ this.ctx.fillStyle = style.pointColor;+ this.ctx.beginPath();+ this.ctx.arc(point.x, point.y, style.pointRadius, 0, Math.PI * 2);+ this.ctx.fill();++ // Draw stroke+ this.ctx.strokeStyle = style.pointStroke;+ this.ctx.lineWidth = 2;+ this.ctx.stroke();++ // Draw label if enabled+ if (style.showLabels && point.id !== undefined) {+ this.drawText(+ point.id.toString(),+ point.x + style.pointRadius + 3,+ point.y - style.pointRadius - 3,+ style.labelFontSize,+ style.labelColor,+ style.labelStroke,+ 2+ );+ }+ }+ }++ /**+ * Generic method to draw segments/connections between points+ */+ protected drawSegments(+ points: Map<string | number, Point2D>,+ segments: Record<string, Segment>,+ style: DrawStyle+ ): void {+ for (const [_, segment] of Object.entries(segments)) {+ const proximalPoint = points.get(segment.proximal);+ const distalPoint = points.get(segment.distal);++ if (!proximalPoint || !distalPoint) continue;+ if (!this.isValidPoint(proximalPoint) || !this.isValidPoint(distalPoint)) continue;++ // Draw line+ this.ctx.strokeStyle = style.lineColor;+ this.ctx.lineWidth = style.lineWidth;+ this.ctx.beginPath();+ this.ctx.moveTo(proximalPoint.x, proximalPoint.y);+ this.ctx.lineTo(distalPoint.x, distalPoint.y);+ this.ctx.stroke();+ }+ }++ /**+ * Generic method to draw a polygon (like ArUco markers)+ */+ protected drawPolygon(+ corners: [number, number][],+ style: DrawStyle,+ fillPoints: boolean = true,+ label?: string+ ): void {+ if (corners.length < 3) return;++ // Draw outline+ this.ctx.strokeStyle = style.lineColor;+ this.ctx.lineWidth = style.lineWidth;+ this.ctx.beginPath();+ this.ctx.moveTo(corners[0][0], corners[0][1]);+ for (let i = 1; i < corners.length; i++) {+ this.ctx.lineTo(corners[i][0], corners[i][1]);+ }+ this.ctx.closePath();+ this.ctx.stroke();++ // Draw corner points if requested+ if (fillPoints) {+ this.ctx.fillStyle = style.pointColor;+ for (const corner of corners) {+ this.ctx.beginPath();+ this.ctx.arc(corner[0], corner[1], style.pointRadius, 0, Math.PI * 2);+ this.ctx.fill();+ }+ }++ // Draw label at center if provided+ if (label) {+ const centerX = corners.reduce((sum, c) => sum + c[0], 0) / corners.length;+ const centerY = corners.reduce((sum, c) => sum + c[1], 0) / corners.length;++ this.drawText(+ label,+ centerX,+ centerY,+ style.labelFontSize * 1.2,+ this.TEXT_COLOR,+ style.pointColor,+ 3,+ 'bold'+ );+ }+ }++ /**+ * Draw text with stroke (outline)+ */+ protected drawText(+ text: string,+ x: number,+ y: number,+ fontSize: number,+ fillColor: string,+ strokeColor: string,+ strokeWidth: number,+ fontWeight: string = 'normal'+ ): void {+ this.ctx.font = `${fontWeight} ${fontSize}px Arial, sans-serif`;+ this.ctx.textBaseline = 'middle';++ // Draw stroke (outline)+ this.ctx.strokeStyle = strokeColor;+ this.ctx.lineWidth = strokeWidth + 2;+ this.ctx.strokeText(text, x, y);++ // Draw fill+ this.ctx.fillStyle = fillColor;+ this.ctx.fillText(text, x, y);+ }++ /**+ * Draw info overlay with metadata+ */+ protected drawInfoOverlay(+ metadata: ObservationMetadata,+ additionalInfo?: string[]+ ): void {+ const lines: string[] = [+ `Frame: ${metadata.frame_number}`,+ ...(additionalInfo || [])+ ];++ // Calculate background size+ const lineHeight = 25;+ const bgHeight = lines.length * lineHeight + 10;+ const bgWidth = 500;++ // Draw background+ this.ctx.fillStyle = this.INFO_BG;+ this.ctx.fillRect(5, 5, bgWidth, bgHeight);++ // Draw lines+ let y = 25;+ for (const line of lines) {+ this.drawText(line, 10, y, 14, this.TEXT_COLOR, this.TEXT_STROKE, 2);+ y += lineHeight;+ }+ }++ /**+ * Check if a point is valid (not NaN or undefined)+ */+ protected isValidPoint(point: Point2D): boolean {+ return (+ point &&+ typeof point.x === 'number' &&+ typeof point.y === 'number' &&+ !isNaN(point.x) &&+ !isNaN(point.y) &&+ isFinite(point.x) &&+ isFinite(point.y)+ );+ }++ /**+ * Create final bitmap from canvas+ */+ protected async createBitmap(sourceBitmap: ImageBitmap): Promise<ImageBitmap> {+ const compositeBitmap = await createImageBitmap(this.offscreenCanvas);+ sourceBitmap.close();+ return compositeBitmap;+ }++ public destroy(): void {+ // Cleanup if needed+ }+}\ No newline at end of filediff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/index.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/index.tsnew file mode 100644index 00000000..8a7dccfd--- /dev/null+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/index.ts@@ -0,0 +1,20 @@+// image-overlay/index.ts - Export all overlay components+export * from './image-overlay-system';+export * from './charuco-types';+export * from './charuco-overlay-renderer';+export * from './overlay-types';+export * from './overlay-renderer-factory';++// Explicitly re-export to avoid duplicate symbol conflicts+// (mediapipe-overlay-renderer re-declares types from mediapipe-types)+export {+ MediapipePointSchema,+ MediapipeMetadataSchema,+ MediapipeOverlaySchema,+ MediapipeOverlayDataMessageSchema,+ type MediapipePoint,+ type MediapipeMetadata,+ type MediapipeObservation,+ type MediapipeOverlayDataMessage+} from './mediapipe-types';+export { MediapipeOverlayRenderer } from './mediapipe-overlay-renderer';\ No newline at end of filediff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/mediapipe-overlay-renderer.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/mediapipe-overlay-renderer.tsnew file mode 100644index 00000000..59d6ddbc--- /dev/null+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/mediapipe-overlay-renderer.ts@@ -0,0 +1,471 @@+// mediapipe-overlay-renderer.ts++import {+ BaseOverlayRenderer,+ DrawStyle,+ Point2D+} from "@/services/server/server-helpers/image-overlay/image-overlay-system";++export interface MediapipePoint {+ name: string;+ x: number;+ y: number;+ z?: number;+ visibility?: number;+}++export interface MediapipeObservation {+ message_type: 'mediapipe_overlay';+ camera_id: string;+ frame_number: number;+ body_points?: MediapipePoint[];+ right_hand_points?: MediapipePoint[];+ left_hand_points?: MediapipePoint[];+ face_points?: MediapipePoint[];+ metadata: {+ image_width: number;+ image_height: number;+ n_body_detected: number;+ n_right_hand_detected: number;+ n_left_hand_detected: number;+ n_face_detected: number;+ };+}++export class MediapipeOverlayRenderer extends BaseOverlayRenderer {+ // Body styles split by side+ private readonly bodyStyleCenter: DrawStyle = {+ pointColor: '#00AA00',+ pointStroke: '#008800',+ pointRadius: 4,+ lineColor: 'rgba(20, 255, 20, 0.6)',+ lineWidth: 2,+ labelColor: '#00AA00',+ labelStroke: this.TEXT_STROKE,+ labelFontSize: 8,+ showLabels: false,+ };++ private readonly bodyStyleRight: DrawStyle = {+ pointColor: '#FF4444',+ pointStroke: '#AA0000',+ pointRadius: 4,+ lineColor: 'rgba(255, 68, 68, 0.6)',+ lineWidth: 2,+ labelColor: '#FF4444',+ labelStroke: this.TEXT_STROKE,+ labelFontSize: 8,+ showLabels: false,+ };++ private readonly bodyStyleLeft: DrawStyle = {+ pointColor: '#4488FF',+ pointStroke: '#0044AA',+ pointRadius: 4,+ lineColor: 'rgba(68, 136, 255, 0.6)',+ lineWidth: 2,+ labelColor: '#4488FF',+ labelStroke: this.TEXT_STROKE,+ labelFontSize: 8,+ showLabels: false,+ };++ private readonly rightHandStyle: DrawStyle = {+ pointColor: '#FF6400',+ pointStroke: '#AA4400',+ pointRadius: 3,+ lineColor: '#FF6400',+ lineWidth: 1.5,+ labelColor: '#FF6400',+ labelStroke: this.TEXT_STROKE,+ labelFontSize: 8,+ showLabels: false,+ };++ private readonly leftHandStyle: DrawStyle = {+ pointColor: '#00AAFF',+ pointStroke: '#0066AA',+ pointRadius: 3,+ lineColor: '#00AAFF',+ lineWidth: 1.5,+ labelColor: '#00AAFF',+ labelStroke: this.TEXT_STROKE,+ labelFontSize: 8,+ showLabels: false,+ };++ private readonly faceStyle: DrawStyle = {+ pointColor: '#FFD700',+ pointStroke: '#AA9900',+ pointRadius: 1,+ lineColor: '#FFD700',+ lineWidth: 1,+ labelColor: '#FFD700',+ labelStroke: this.TEXT_STROKE,+ labelFontSize: 6,+ showLabels: false,+ };++ /**+ * Composite mediapipe overlay onto frame+ */+ public async compositeFrame(+ sourceBitmap: ImageBitmap,+ observation: MediapipeObservation | null,+ ): Promise<ImageBitmap> {+ this.prepareCanvas(sourceBitmap);++ if (observation) {+ this.drawMediapipeOverlay(observation);+ }++ return this.createBitmap(sourceBitmap);+ }++ private drawMediapipeOverlay(observation: MediapipeObservation): void {+ this.ctx.save();++ // Draw each aspect in order (face behind, body in middle, hands on top)+ if (observation.face_points) {+ this.drawFaceAspect(observation.face_points);+ }++ if (observation.body_points) {+ this.drawBodyAspect(observation.body_points);+ }++ if (observation.right_hand_points) {+ this.drawHandAspect(observation.right_hand_points, 'right_hand', this.rightHandStyle);+ }++ if (observation.left_hand_points) {+ this.drawHandAspect(observation.left_hand_points, 'left_hand', this.leftHandStyle);+ }++ // Draw info overlay+ this.drawMediapipeInfo(observation);++ this.ctx.restore();+ }++ private getBodySide(name: string): 'left' | 'right' | 'center' {+ if (name.includes('left')) return 'left';+ if (name.includes('right')) return 'right';+ return 'center';+ }++ private getBodyStyleForSide(side: 'left' | 'right' | 'center'): DrawStyle {+ switch (side) {+ case 'left': return this.bodyStyleLeft;+ case 'right': return this.bodyStyleRight;+ case 'center': return this.bodyStyleCenter;+ }+ }++ private drawBodyAspect(points: MediapipePoint[]): void {+ if (points.length === 0) return;++ // Create point map for segment drawing+ const pointMap = new Map<string | number, Point2D>();+ const leftPoints: Point2D[] = [];+ const rightPoints: Point2D[] = [];+ const centerPoints: Point2D[] = [];++ for (const point of points) {+ const point2D: Point2D = {+ x: point.x,+ y: point.y,+ id: point.name,+ visibility: point.visibility,+ };+ pointMap.set(point.name, point2D);++ const side = this.getBodySide(point.name);+ if (side === 'left') leftPoints.push(point2D);+ else if (side === 'right') rightPoints.push(point2D);+ else centerPoints.push(point2D);+ }++ // Draw body skeleton connections (behind points)+ this.drawBodyConnections(pointMap);++ // Draw points on top, color-coded by side+ this.drawPoints(leftPoints, this.bodyStyleLeft);+ this.drawPoints(rightPoints, this.bodyStyleRight);+ this.drawPoints(centerPoints, this.bodyStyleCenter);++ // Highlight key points with labels+ this.drawKeyBodyPoints(pointMap);+ }++ private drawKeyBodyPoints(pointMap: Map<string | number, Point2D>): void {+ // Label only important landmarks for clarity+ const keyPoints = ['body.nose', 'body.left_shoulder', 'body.right_shoulder', 'body.left_hip', 'body.right_hip'];++ for (const key of keyPoints) {+ const point = pointMap.get(key);+ if (point && this.isValidPoint(point)) {+ const style = this.getBodyStyleForSide(this.getBodySide(key));+ this.drawText(+ key.replace('body.', '').replace('_', ' '),+ point.x + 5,+ point.y - 5,+ 8,+ style.labelColor,+ style.labelStroke,+ 1+ );+ }+ }+ }++ private drawBodyConnections(+ pointMap: Map<string | number, Point2D>,+ ): void {+ // Connections grouped by side for color coding+ const centerConnections = [+ ['body.nose', 'body.left_eye_inner'],+ ['body.nose', 'body.right_eye_inner'],+ ['body.mouth_left', 'body.mouth_right'],+ ['body.left_shoulder', 'body.right_shoulder'],+ ['body.left_hip', 'body.right_hip'],+ ];++ const leftConnections = [+ ['body.left_eye_inner', 'body.left_eye'],+ ['body.left_eye', 'body.left_eye_outer'],+ ['body.left_eye_outer', 'body.left_ear'],+ ['body.left_shoulder', 'body.left_hip'],+ ['body.left_shoulder', 'body.left_elbow'],+ ['body.left_elbow', 'body.left_wrist'],+ ['body.left_wrist', 'body.left_pinky'],+ ['body.left_wrist', 'body.left_index'],+ ['body.left_wrist', 'body.left_thumb'],+ ['body.left_index', 'body.left_pinky'],+ ['body.left_hip', 'body.left_knee'],+ ['body.left_knee', 'body.left_ankle'],+ ['body.left_ankle', 'body.left_heel'],+ ['body.left_heel', 'body.left_foot_index'],+ ['body.left_ankle', 'body.left_foot_index'],+ ];++ const rightConnections = [+ ['body.right_eye_inner', 'body.right_eye'],+ ['body.right_eye', 'body.right_eye_outer'],+ ['body.right_eye_outer', 'body.right_ear'],+ ['body.right_shoulder', 'body.right_hip'],+ ['body.right_shoulder', 'body.right_elbow'],+ ['body.right_elbow', 'body.right_wrist'],+ ['body.right_wrist', 'body.right_pinky'],+ ['body.right_wrist', 'body.right_index'],+ ['body.right_wrist', 'body.right_thumb'],+ ['body.right_index', 'body.right_pinky'],+ ['body.right_hip', 'body.right_knee'],+ ['body.right_knee', 'body.right_ankle'],+ ['body.right_ankle', 'body.right_heel'],+ ['body.right_heel', 'body.right_foot_index'],+ ['body.right_ankle', 'body.right_foot_index'],+ ];++ const groups: [string[][], DrawStyle][] = [+ [centerConnections, this.bodyStyleCenter],+ [leftConnections, this.bodyStyleLeft],+ [rightConnections, this.bodyStyleRight],+ ];++ for (const [connections, style] of groups) {+ for (const [start, end] of connections) {+ const startPoint = pointMap.get(start);+ const endPoint = pointMap.get(end);++ if (startPoint && endPoint && this.isValidPoint(startPoint) && this.isValidPoint(endPoint)) {+ this.ctx.strokeStyle = style.lineColor;+ this.ctx.lineWidth = style.lineWidth;+ this.ctx.beginPath();+ this.ctx.moveTo(startPoint.x, startPoint.y);+ this.ctx.lineTo(endPoint.x, endPoint.y);+ this.ctx.stroke();+ }+ }+ }+ }++ private drawHandAspect(+ points: MediapipePoint[],+ aspectName: string,+ style: DrawStyle+ ): void {+ if (points.length === 0) return;++ const pointMap = new Map<string | number, Point2D>();+ const point2DArray: Point2D[] = [];++ for (const point of points) {+ const point2D: Point2D = {+ x: point.x,+ y: point.y,+ id: point.name,+ visibility: point.visibility,+ };+ pointMap.set(point.name, point2D);+ point2DArray.push(point2D);+ }++ // Draw hand skeleton connections+ this.drawHandConnections(pointMap, style);++ // Draw points+ this.drawPoints(point2DArray, style);+ }++ private drawHandConnections(+ pointMap: Map<string | number, Point2D>,+ style: DrawStyle+ ): void {+ // Define hand connections (MediaPipe hand landmark connections)+ const handConnections = [+ // Thumb+ ['wrist', 'thumb_cmc'],+ ['thumb_cmc', 'thumb_mcp'],+ ['thumb_mcp', 'thumb_ip'],+ ['thumb_ip', 'thumb_tip'],+ // Index finger+ ['wrist', 'index_finger_mcp'],+ ['index_finger_mcp', 'index_finger_pip'],+ ['index_finger_pip', 'index_finger_dip'],+ ['index_finger_dip', 'index_finger_tip'],+ // Middle finger+ ['wrist', 'middle_finger_mcp'],+ ['middle_finger_mcp', 'middle_finger_pip'],+ ['middle_finger_pip', 'middle_finger_dip'],+ ['middle_finger_dip', 'middle_finger_tip'],+ // Ring finger+ ['wrist', 'ring_finger_mcp'],+ ['ring_finger_mcp', 'ring_finger_pip'],+ ['ring_finger_pip', 'ring_finger_dip'],+ ['ring_finger_dip', 'ring_finger_tip'],+ // Pinky+ ['wrist', 'pinky_mcp'],+ ['pinky_mcp', 'pinky_pip'],+ ['pinky_pip', 'pinky_dip'],+ ['pinky_dip', 'pinky_tip'],+ // Palm connections+ ['index_finger_mcp', 'middle_finger_mcp'],+ ['middle_finger_mcp', 'ring_finger_mcp'],+ ['ring_finger_mcp', 'pinky_mcp'],+ ];++ for (const [start, end] of handConnections) {+ const startPoint = pointMap.get(start);+ const endPoint = pointMap.get(end);++ if (startPoint && endPoint && this.isValidPoint(startPoint) && this.isValidPoint(endPoint)) {+ this.ctx.strokeStyle = style.lineColor;+ this.ctx.lineWidth = style.lineWidth;+ this.ctx.beginPath();+ this.ctx.moveTo(startPoint.x, startPoint.y);+ this.ctx.lineTo(endPoint.x, endPoint.y);+ this.ctx.stroke();+ }+ }+ }++ private drawFaceAspect(points: MediapipePoint[]): void {+ if (points.length === 0) return;++ // For face, we just draw the points as a mesh without connections+ // (too many points to draw connections clearly)+ const point2DArray: Point2D[] = points.map(point => ({+ x: point.x,+ y: point.y,+ id: point.name,+ visibility: point.visibility,+ }));++ this.drawPoints(point2DArray, this.faceStyle);++ // Draw face contour highlights for key features+ this.drawFaceContour(points);+ }++ private drawFaceContour(points: MediapipePoint[]): void {+ // Draw simplified face contour for better visibility+ // This would use the face contour indices from the model info+ // For now, just highlight key face points+ const keyFacePoints = points.filter(p =>+ p.name.includes('eye') ||+ p.name.includes('mouth') ||+ p.name.includes('nose')+ );++ const highlightStyle = {+ ...this.faceStyle,+ pointRadius: 2,+ pointColor: '#FFFF00',+ };++ this.drawPoints(+ keyFacePoints.map(p => ({+ x: p.x,+ y: p.y,+ id: p.name,+ visibility: p.visibility,+ })),+ highlightStyle+ );+ }++ private drawMediapipeInfo(observation: MediapipeObservation): void {+ const { metadata, frame_number } = observation;++ // Build detection status+ const detections: string[] = [];+ if (metadata.n_body_detected > 0) detections.push('Body');+ if (metadata.n_right_hand_detected > 0) detections.push('R-Hand');+ if (metadata.n_left_hand_detected > 0) detections.push('L-Hand');+ if (metadata.n_face_detected > 0) detections.push('Face');++ const statusText = detections.length > 0+ ? `β Detected: ${detections.join(', ')}`+ : 'β No detection';++ const statusColor = detections.length > 0 ? '#00FF00' : '#FF4444';++ // Draw info background+ this.ctx.fillStyle = this.INFO_BG;+ this.ctx.fillRect(5, 5, 500, 85);++ // Draw frame info+ this.drawText(+ `Frame: ${frame_number}`,+ 10,+ 25,+ 14,+ this.TEXT_COLOR,+ this.TEXT_STROKE,+ 2+ );++ // Draw detection counts+ this.drawText(+ `Body: ${metadata.n_body_detected} | Hands: ${metadata.n_right_hand_detected + metadata.n_left_hand_detected} | Face: ${metadata.n_face_detected > 0 ? 'β' : 'β'}`,+ 10,+ 50,+ 14,+ this.TEXT_COLOR,+ this.TEXT_STROKE,+ 2+ );++ // Draw status+ this.drawText(+ statusText,+ 10,+ 75,+ 16,+ statusColor,+ this.TEXT_STROKE,+ 2+ );+ }+}\ No newline at end of filediff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/mediapipe-types.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/mediapipe-types.tsnew file mode 100644index 00000000..f103f700--- /dev/null+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/mediapipe-types.ts@@ -0,0 +1,41 @@+import z from "zod";++// Individual point schema+export const MediapipePointSchema = z.object({+ name: z.string(),+ x: z.number(),+ y: z.number(),+ z: z.number(),+ visibility: z.number(),+});++// Metadata schema+export const MediapipeMetadataSchema = z.object({+ n_body_detected: z.number(),+ n_right_hand_detected: z.number(),+ n_left_hand_detected: z.number(),+ n_face_detected: z.number(),+ image_width: z.number(),+ image_height: z.number(),+});++// Single camera observation schema+export const MediapipeOverlaySchema = z.object({+ message_type: z.literal("mediapipe_overlay"),+ camera_id: z.string(),+ frame_number: z.number(),+ body_points: z.array(MediapipePointSchema),+ right_hand_points: z.array(MediapipePointSchema),+ left_hand_points: z.array(MediapipePointSchema),+ face_points: z.array(MediapipePointSchema),+ metadata: MediapipeMetadataSchema,+});++// Multi-camera message schema (matches CharucoOverlayDataMessage structure)+export const MediapipeOverlayDataMessageSchema = z.record(z.string(), MediapipeOverlaySchema);++// Type exports+export type MediapipePoint = z.infer<typeof MediapipePointSchema>;+export type MediapipeMetadata = z.infer<typeof MediapipeMetadataSchema>;+export type MediapipeObservation = z.infer<typeof MediapipeOverlaySchema>;+export type MediapipeOverlayDataMessage = z.infer<typeof MediapipeOverlayDataMessageSchema>;\ No newline at end of filediff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/overlay-renderer-factory.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/overlay-renderer-factory.tsnew file mode 100644index 00000000..060463b3--- /dev/null+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/overlay-renderer-factory.ts@@ -0,0 +1,145 @@+// overlay-renderer-factory.ts+import { CharucoOverlayRenderer } from './charuco-overlay-renderer';+import { MediapipeOverlayRenderer, MediapipeObservation } from './mediapipe-overlay-renderer';+import {BaseOverlayRenderer, ModelInfo} from "@/services/server/server-helpers/image-overlay/image-overlay-system";+import {CharucoObservation} from "@/services/server/server-helpers/image-overlay/charuco-types";++export type ObservationType = 'charuco_overlay' | 'mediapipe_overlay' | 'rtmpose_overlay';+++/**+ * Factory for creating appropriate overlay renderers based on observation type+ */+export class OverlayRendererFactory {+ private static rendererCache: Map<ObservationType, BaseOverlayRenderer> = new Map();+ private static modelInfoCache: Map<string, ModelInfo> = new Map();++ /**+ * Get or create a renderer for the specified observation type+ */+ public static getRenderer(type: ObservationType): BaseOverlayRenderer {+ // Check cache first+ let renderer = this.rendererCache.get(type);+ if (renderer) {+ return renderer;+ }++ // Create new renderer based on type+ switch (type) {+ case 'charuco_overlay':+ renderer = new CharucoOverlayRenderer();+ break;+ case 'mediapipe_overlay':+ renderer = new MediapipeOverlayRenderer();+ break;+ case 'rtmpose_overlay':+ // Future: implement RTMPoseOverlayRenderer+ throw new Error('RTMPose overlay not yet implemented');+ default:+ throw new Error(`Unknown observation type: ${type}`);+ }++ this.rendererCache.set(type, renderer);+ return renderer;+ }++ /**+ * Set model info for a specific tracker type+ */+ public static setModelInfo(trackerName: string, modelInfo: ModelInfo): void {+ this.modelInfoCache.set(trackerName, modelInfo);++ // Update all renderers that might use this model info+ for (const [type, renderer] of this.rendererCache) {+ if (this.shouldUseModelInfo(type, trackerName)) {+ renderer.setModelInfo(modelInfo);+ }+ }+ }++ /**+ * Clear all cached renderers+ */+ public static clearCache(): void {+ for (const renderer of this.rendererCache.values()) {+ renderer.destroy();+ }+ this.rendererCache.clear();+ this.modelInfoCache.clear();+ }++ /**+ * Determine if a renderer type should use a specific model info+ */+ private static shouldUseModelInfo(+ rendererType: ObservationType,+ trackerName: string+ ): boolean {+ const mapping: Record<ObservationType, string[]> = {+ 'charuco_overlay': ['CharucoTracker'],+ 'mediapipe_overlay': ['MediapipeHolisticTracker', 'MediapipeTracker'],+ 'rtmpose_overlay': ['RTMPoseTracker'],+ };++ return mapping[rendererType]?.includes(trackerName) || false;+ }+}++/**+ * Manager for handling overlay rendering across multiple cameras.+ *+ * Composites both charuco and mediapipe overlays sequentially onto each+ * frame: source β charuco β mediapipe. Each renderer draws on top of+ * the previous result rather than clearing and redrawing from scratch.+ */+export class OverlayManager {+ private charucoRenderer: CharucoOverlayRenderer = new CharucoOverlayRenderer();+ private mediapipeRenderer: MediapipeOverlayRenderer = new MediapipeOverlayRenderer();++ /**+ * Composite both overlay types onto a single frame, chained sequentially.+ * The charuco overlay is drawn first, then mediapipe on top of the result.+ */+ public async processFrame(+ cameraId: string,+ sourceBitmap: ImageBitmap,+ charucoObservation: CharucoObservation | null,+ mediapipeObservation: MediapipeObservation | null,+ ): Promise<ImageBitmap> {+ let currentBitmap = sourceBitmap;++ if (charucoObservation) {+ currentBitmap = await this.charucoRenderer.compositeFrame(+ currentBitmap,+ charucoObservation,+ );+ }++ if (mediapipeObservation) {+ currentBitmap = await this.mediapipeRenderer.compositeFrame(+ currentBitmap,+ mediapipeObservation,+ );+ }++ return currentBitmap;+ }++ /**+ * Set model info on both renderers+ */+ public setModelInfo(modelInfo: ModelInfo): void {+ this.charucoRenderer.setModelInfo(modelInfo);+ this.mediapipeRenderer.setModelInfo(modelInfo);+ }++ /**+ * Clear all renderers+ */+ public clearAll(): void {+ this.charucoRenderer.destroy();+ this.mediapipeRenderer.destroy();+ this.charucoRenderer = new CharucoOverlayRenderer();+ this.mediapipeRenderer = new MediapipeOverlayRenderer();+ }+}\ No newline at end of filediff --git a/freemocap-ui/src/services/server/server-helpers/image-overlay/overlay-types.ts b/freemocap-ui/src/services/server/server-helpers/image-overlay/overlay-types.tsnew file mode 100644index 00000000..7bf5c2e5--- /dev/null+++ b/freemocap-ui/src/services/server/server-helpers/image-overlay/overlay-types.ts@@ -0,0 +1,17 @@+import {CharucoObservation, CharucoOverlayDataMessage} from "@/services/server/server-helpers/image-overlay/charuco-types";+import {MediapipeObservation, MediapipeOverlayDataMessage} from "@/services/server/server-helpers/image-overlay/mediapipe-types";++// Union type for all observation data messages+export type ObservationDataMessage = CharucoOverlayDataMessage | MediapipeOverlayDataMessage;++// Union type for individual observations+export type Observation = CharucoObservation | MediapipeObservation;++// Type guard functions+export function isCharucoObservation(obs: Observation): obs is CharucoObservation {+ return obs.message_type === 'charuco_overlay';+}++export function isMediapipeObservation(obs: Observation): obs is MediapipeObservation {+ return obs.message_type === 'mediapipe_overlay';+}\ No newline at end of filediff --git a/freemocap-ui/src/services/server/server-helpers/websocket-message-types.ts b/freemocap-ui/src/services/server/server-helpers/websocket-message-types.tsnew file mode 100644index 00000000..59130c97--- /dev/null+++ b/freemocap-ui/src/services/server/server-helpers/websocket-message-types.ts@@ -0,0 +1,101 @@+import {+ CharucoOverlayDataMessage,+ CharucoOverlayDataMessageSchema,+} from "@/services/server/server-helpers/image-overlay/charuco-types";+import {+ MediapipeOverlayDataMessage,+ MediapipeOverlayDataMessageSchema,+} from "@/services/server/server-helpers/image-overlay/mediapipe-types";+import {ModelInfo} from "@/services/server/server-helpers/image-overlay/image-overlay-system";+import {OverlayRendererFactory} from "@/services/server/server-helpers/image-overlay/overlay-renderer-factory";++export interface FramerateUpdateMessage {+ message_type: 'framerate_update';+ camera_group_id: string;+ backend_framerate: {+ mean: number;+ std: number;+ min: number;+ max: number;+ median: number;+ recent: number[];+ };+ frontend_framerate: {+ mean: number;+ std: number;+ min: number;+ max: number;+ median: number;+ recent: number[];+ };+}++export interface LogRecord {+ message_type: 'log_record';+ levelname: string;+ message: string;+ timestamp: string;+}++export interface SettingsStateMessage {+ message_type: 'settings/state';+ settings: Record<string, unknown>;+ version: number;+}++export function arraysEqual(a: string[], b: string[]): boolean {+ if (a.length !== b.length) return false;+ const sortedA = [...a].sort();+ const sortedB = [...b].sort();+ return sortedA.every((val, idx) => val === sortedB[idx]);+}++export function isLogRecord(data: unknown): data is LogRecord {+ return (+ typeof data === 'object' &&+ data !== null &&+ (data as Record<string, unknown>).message_type === 'log_record' &&+ typeof (data as Record<string, unknown>).levelname === 'string' &&+ typeof (data as Record<string, unknown>).message === 'string'+ );+}++export function isSettingsStateMessage(data: unknown): data is SettingsStateMessage {+ return (+ typeof data === 'object' &&+ data !== null &&+ (data as Record<string, unknown>).message_type === 'settings/state' &&+ typeof (data as Record<string, unknown>).settings === 'object' &&+ (data as Record<string, unknown>).settings !== null &&+ typeof (data as Record<string, unknown>).version === 'number'+ );+}++export function isFramerateUpdate(data: unknown): data is FramerateUpdateMessage {+ const d = data as Record<string, unknown>;+ return (+ typeof data === 'object' &&+ data !== null &&+ d.message_type === 'framerate_update' &&+ typeof d.camera_group_id === 'string' &&+ typeof d.backend_framerate === 'object' &&+ d.backend_framerate !== null &&+ typeof d.frontend_framerate === 'object' &&+ d.frontend_framerate !== null+ );+}++export function isCharucoOverlayDataMessage(data: unknown): data is CharucoOverlayDataMessage {+ const result = CharucoOverlayDataMessageSchema.safeParse(data);+ return result.success;+}++export function isMediapipeOverlayDataMessage(data: unknown): data is MediapipeOverlayDataMessage {+ const result = MediapipeOverlayDataMessageSchema.safeParse(data);+ return result.success;+}++export function handleModelInfoUpdate(modelInfo: ModelInfo): void {+ console.log(`Received model info for tracker: ${modelInfo.tracker_name}`);+ OverlayRendererFactory.setModelInfo(modelInfo.tracker_name, modelInfo);+}\ No newline at end of filediff --git a/freemocap/api/websocket/websocket_server.py b/freemocap/api/websocket/websocket_server.pyindex 5e1169a6..5f8b238a 100644--- a/freemocap/api/websocket/websocket_server.py+++ b/freemocap/api/websocket/websocket_server.py@@ -189,6 +189,10 @@ class WebsocketServer:point_name: tracked_point.model_dump()for point_name, tracked_point in frontend_payload.tracked_points3d.items()}+ for point_name, point in points3d_dict.items():+ point['x'] *= 0.001+ point['y'] *= 0.001+ point['z'] *= 0.001async with self._send_lock:await self.websocket.send_json({"tracked_points3d": points3d_dict})@@ -316,4 +320,4 @@ class WebsocketServer:self._websocket_should_continue = Falseraisefinally:- logger.info("Ending client message handler...")\ No newline at end of file+ logger.info("Ending client message handler...")diff --git a/shared/freemocap-logo/freemocap-favicon.ico b/shared/freemocap-logo/freemocap-favicon.iconew file mode 100644index 00000000..ade9aa25Binary files /dev/null and b/shared/freemocap-logo/freemocap-favicon.ico differdiff --git a/shared/freemocap-logo/freemocap-logo.ico b/shared/freemocap-logo/freemocap-logo.icodeleted file mode 100644index f5271037..00000000Binary files a/shared/freemocap-logo/freemocap-logo.ico and /dev/null differ