Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

hell yea! Python images making it to React view :D

jonmatthis147ca9bddb85
diff --git a/freemocap-ui/src/context/WebSocketContext.tsx b/freemocap-ui/src/context/WebSocketContext.tsx
index 7ca5059b..d19344d0 100644
--- a/freemocap-ui/src/context/WebSocketContext.tsx
+++ b/freemocap-ui/src/context/WebSocketContext.tsx
@@ -1,9 +1,11 @@
import React, {createContext, useContext, ReactNode} from "react";
import {useWebSocket} from "@/hooks/useWebSocket";
+import {FrontendFramePayloadSchema} from "@/models/FrontendImagePayload";
+import {z} from "zod";
interface WebSocketContextProps {
isConnected: boolean;
- messages: string[];
+ latestFrontendPayload: z.infer<typeof FrontendFramePayloadSchema> | null;
connect: () => void;
disconnect: () => void;
}
@@ -14,10 +16,10 @@ interface WebSocketProviderProps {
const WebSocketContext = createContext<WebSocketContextProps | undefined>(undefined);
export const WebSocketProvider: React.FC<WebSocketProviderProps> = ({url, children}) => {
- const {isConnected, messages, connect, disconnect} = useWebSocket(url);
+ const {isConnected, latestFrontendPayload, connect, disconnect} = useWebSocket(url);
return(
- <WebSocketContext.Provider value={{isConnected, messages, connect, disconnect}}>
+ <WebSocketContext.Provider value={{isConnected, latestFrontendPayload, connect, disconnect}}>
{children}
</WebSocketContext.Provider>
)
diff --git a/freemocap-ui/src/hooks/useWebSocket.ts b/freemocap-ui/src/hooks/useWebSocket.ts
index badd3337..3e30cf4b 100644
--- a/freemocap-ui/src/hooks/useWebSocket.ts
+++ b/freemocap-ui/src/hooks/useWebSocket.ts
@@ -1,13 +1,42 @@
import {useCallback, useEffect, useState} from 'react';
+import {z} from 'zod';
+import {FrontendFramePayloadSchema} from "@/models/FrontendImagePayload";
const MAX_RECONNECT_ATTEMPTS = 20;
export const useWebSocket = (wsUrl: string) => {
const [isConnected, setIsConnected] = useState(false);
- const [messages, setMessages] = useState<string[]>([]);
+ const [latestFrontendPayload, setLatestFrontendPayload] = useState<z.infer<typeof FrontendFramePayloadSchema> | null>(null);
const [websocket, setWebSocket] = useState<WebSocket | null>(null);
const [connectAttempt, setConnectAttempt] = useState(0);
+ const handleIncomingMessage = (data: Blob | string) => {
+ if (typeof data === 'string') {
+ parseAndValidateMessage(data);
+ } else if (data instanceof Blob) {
+ // If data is a Blob, convert it to text
+ data.text().then(text => {
+ parseAndValidateMessage(text);
+ }).catch(error => {
+ console.error('Error reading Blob data:', error);
+ });
+ }
+ };
+
+ const parseAndValidateMessage = (data: string) => {
+ try {
+ const parsedData = JSON.parse(data);
+ const frontendImagePayload = FrontendFramePayloadSchema.parse(parsedData);
+ setLatestFrontendPayload(frontendImagePayload);
+ } catch (e) {
+ if (e instanceof z.ZodError) {
+ console.error('Validation failed with errors:', e.errors);
+ } else {
+ console.error('Error parsing message data:', e);
+ }
+ }
+ };
+
const connect = useCallback(() => {
if (websocket && websocket.readyState !== WebSocket.CLOSED) {
return;
@@ -31,7 +60,7 @@ export const useWebSocket = (wsUrl: string) => {
ws.onmessage = (event) => {
console.log('Websocket message received with length: ', event.data.length);
- setMessages(prevMessages => [...prevMessages, event.data]);
+ handleIncomingMessage(event.data);
};
ws.onerror = (error) => {
@@ -59,5 +88,5 @@ export const useWebSocket = (wsUrl: string) => {
};
}, [connect]);
- return {isConnected, messages, connect, disconnect};
+ return {isConnected, latestFrontendPayload, connect, disconnect};
};
diff --git a/freemocap-ui/src/models/FrontendImagePayload.ts b/freemocap-ui/src/models/FrontendImagePayload.ts
new file mode 100644
index 00000000..71de05df
--- /dev/null
+++ b/freemocap-ui/src/models/FrontendImagePayload.ts
@@ -0,0 +1,10 @@
+import { z } from "zod";
+
+export const FrontendFramePayloadSchema = z.object({
+ type: z.literal('FrontendFramePayload'),
+ jpeg_images: z.record(z.string().nullable()), // Allow dictionary-like structure
+ camera_configs: z.record(z.any()), // Allow any shape for camera_configs
+ multi_frame_metadata: z.any(), // Allow any value for multi_frame_metadata
+ utc_ns_to_perf_ns: z.any(), // Allow any value for utc_ns_to_perf_ns
+ multi_frame_number: z.number().int().default(0),
+});
diff --git a/freemocap-ui/src/views/PythonCamerasView.tsx b/freemocap-ui/src/views/PythonCamerasView.tsx
index f4970396..061b757b 100644
--- a/freemocap-ui/src/views/PythonCamerasView.tsx
+++ b/freemocap-ui/src/views/PythonCamerasView.tsx
@@ -1,12 +1,28 @@
import {Box, Button} from "@mui/material";
import React from "react";
import {ConnectToCamerasButton} from "@/components/ConnectToCamerasButton";
+import {useWebSocketContext} from "@/context/WebSocketContext";
export const PythonCamerasView = () => {
+ const { latestFrontendPayload } = useWebSocketContext();
return (
- <Box sx={{display: "flex", flexDirection: "column"}}>
- <ConnectToCamerasButton/>
+ <Box sx={{ display: "flex", flexDirection: "column" }}>
+ <ConnectToCamerasButton />
+ {latestFrontendPayload && latestFrontendPayload.jpeg_images && (
+ <Box sx={{ display: "flex", flexDirection: "row", flexWrap: "wrap" }}>
+ {Object.entries(latestFrontendPayload.jpeg_images).map(([cameraId, base64Image]) => (
+ base64Image ? (
+ <img
+ key={cameraId}
+ src={`data:image/jpeg;base64,${base64Image}`}
+ alt={`Camera ${cameraId}`}
+ style={{ width: "200px", height: "auto", margin: "10px" }} // Adjust size as needed
+ />
+ ) : null
+ ))}
+ </Box>
+ )}
</Box>
);
}
diff --git a/freemocap-ui/src/views/WebsocketConnection.tsx b/freemocap-ui/src/views/WebsocketConnection.tsx
index 2a22ebea..47a6666f 100644
--- a/freemocap-ui/src/views/WebsocketConnection.tsx
+++ b/freemocap-ui/src/views/WebsocketConnection.tsx
@@ -3,7 +3,7 @@ import {Box} from '@mui/material';
import {useWebSocketContext} from '@/context/WebSocketContext';
const WebsocketConnection = () => {
- const {isConnected, messages} = useWebSocketContext();
+ const {isConnected} = useWebSocketContext();
return (
<Box sx={{display: 'flex', flexDirection: 'column'}}>