Free Motion Capture for Everyone πβ¨
TypeScript
getting started with the react refactor
jonmatthis47daa2c4545c
diff --git a/freemocap-ui/src/hooks/use-frame-capture.tsx b/freemocap-ui/src/hooks/use-frame-capture.tsxdeleted file mode 100644index f1f05510..00000000--- a/freemocap-ui/src/hooks/use-frame-capture.tsx+++ /dev/null@@ -1,27 +0,0 @@-import {useEffect, useState} from "react";-import {CaptureType, FrameCapture, OnMessageHandler} from "../services/frame-capture";--export const useFrameCapture = ( captureType: CaptureType, port:number, onMessage?: OnMessageHandler): [FrameCapture, {[key: string]: string}] => {- const [webcam_id] = useState<string>("0");- const [frameCapture,] = useState(() => new FrameCapture( captureType, port));-- const [data, setDataUrls] = useState<{[key: string]: string}>({});-- useEffect(() => {- frameCapture.start_frame_capture(async (ev: MessageEvent<Blob>, data_urls: { [key: string]: string }) => {- setDataUrls(data_urls as {[key: string]: string});- if (onMessage) {- await onMessage(ev, data_urls);- }- });- return () => {- frameCapture.cleanup()- }- }, [frameCapture, onMessage, webcam_id]);-- window.onbeforeunload = () => {- frameCapture.cleanup()- }-- return [frameCapture, data];-}diff --git a/freemocap-ui/src/hooks/use-websocket.tsx b/freemocap-ui/src/hooks/use-websocket.tsxnew file mode 100644index 00000000..9daf4abf--- /dev/null+++ b/freemocap-ui/src/hooks/use-websocket.tsx@@ -0,0 +1,30 @@+import {useEffect, useState} from "react";+import {CaptureType, WebsocketConnection, OnMessageHandler} from "../services/websocket-connection";++export const useWebsocket = (captureType: CaptureType,+ port: number,+ onMessage?: OnMessageHandler): [WebsocketConnection, { [key: string]: string }] => {++ const [webcam_id] = useState<string>("0");+ const [websocketConnection,] = useState(() => new WebsocketConnection(captureType, port));++ const [dataUrls, setDataUrls] = useState<{ [key: string]: string }>({});++ useEffect(() => {+ websocketConnection.connect_to_cameras(async (ev: MessageEvent<Blob>, data_urls: { [key: string]: string }) => {+ setDataUrls(data_urls as { [key: string]: string });+ if (onMessage) {+ await onMessage(ev, data_urls);+ }+ });+ return () => {+ websocketConnection.cleanup()+ }+ }, [websocketConnection, onMessage, webcam_id]);++ window.onbeforeunload = () => {+ websocketConnection.cleanup()+ }++ return [websocketConnection, dataUrls];+}diff --git a/freemocap-ui/src/layout/routing/router.tsx b/freemocap-ui/src/layout/routing/router.tsxindex 36df547b..c7f1005f 100644--- a/freemocap-ui/src/layout/routing/router.tsx+++ b/freemocap-ui/src/layout/routing/router.tsx@@ -1,11 +1,11 @@import React from 'react';import {Route, Routes} from "react-router-dom";import {ConfigView} from "@/views/Config";-import {WebcamJonTest} from "@/views/WebcamJonTest";+import {BrowserCamerasView} from "@/views/BrowserCamerasView";import {PythonToJsTest} from "@/views/PythonToJsTest";import {BoardDetection} from "@/views/BoardDetection";import {SkeletonDetection} from "@/views/SkeletonDetection";-import {ShowCameras} from "@/views/ShowCameras";+import {PythonCamerasView} from "@/views/PythonCamerasView";import {DefaultView} from "@/views/Default";export const Router = () => {@@ -14,11 +14,11 @@ export const Router = () => {<Route path={'/'} element={<React.Fragment/>}/><Route path={'/default'} element={<DefaultView/>}/><Route path={'/config'} element={<ConfigView/>}/>- <Route path={'/jontestplayground'} element={<WebcamJonTest/>}/>+ <Route path={'/jontestplayground'} element={<BrowserCamerasView/>}/><Route path={'/pythonToJs'} element={<PythonToJsTest/>}/><Route path={'/charuco_board_detection'} element={<BoardDetection/>}/><Route path={'/skeleton_detection'} element={<SkeletonDetection/>}/>- <Route path={'/show_cameras'} element={<ShowCameras/>}/>+ <Route path={'/show_cameras'} element={<PythonCamerasView/>}/></Routes>)}diff --git a/freemocap-ui/src/services/cam.ts b/freemocap-ui/src/services/detectCameraDevices.tssimilarity index 84%rename from freemocap-ui/src/services/cam.tsrename to freemocap-ui/src/services/detectCameraDevices.tsindex 443e9ee9..e216c3c9 100644--- a/freemocap-ui/src/services/cam.ts+++ b/freemocap-ui/src/services/detectCameraDevices.ts@@ -1,5 +1,5 @@-export class BrowserCam {- public async findAllCameras(filterVirtual: boolean = false) {+export class AvailableCameraDevices {+ public async findAllCameras(filterVirtual: boolean = true) {const devices = await navigator.mediaDevices.enumerateDevices();const cameras = devices.filter(({kind, label}) => kind === "videoinput");console.log(`Found ${cameras.length} cameras - ${cameras.map(({label}) => label).join("\n") }`);diff --git a/freemocap-ui/src/services/frame-capture.tsx b/freemocap-ui/src/services/websocket-connection.tsxsimilarity index 71%rename from freemocap-ui/src/services/frame-capture.tsxrename to freemocap-ui/src/services/websocket-connection.tsxindex c32d3270..66c6db83 100644--- a/freemocap-ui/src/services/frame-capture.tsx+++ b/freemocap-ui/src/services/websocket-connection.tsx@@ -1,19 +1,19 @@export type OnMessageHandler = (ev: MessageEvent<Blob>, data_urls: { [key: string]: string }) => Promise<void>;export enum CaptureType {- BoardDetection = "board_detection", SkeletonDetection = "skeleton_detection", ConnectCameras = "connect"+ ConnectCameras = "connect"}-export class FrameCapture {+export class WebsocketConnection {private _ws_connection!: WebSocket;- private readonly _host: string;- private readonly _base_host: string;+ private readonly _host_url: string;+ private readonly _base_host_url: string;- constructor(private readonly _captureType: CaptureType = CaptureType.BoardDetection,+ constructor(private readonly _captureType: CaptureType = CaptureType.ConnectCameras,private readonly _port: number = 8005) {- this._base_host = `ws://localhost:${_port}/websocket`;- this._host = `${this._base_host}/${this._captureType}`- console.log(`FrameCapture : ${this._host}`)+ this._base_host_url = `ws://localhost:${_port}/websocket`;+ this._host_url = `${this._base_host_url}/${this._captureType}`+ console.log(`WebsocketConnection.constructor: ${this._host_url}`);}private _current_data_url!: string;@@ -25,10 +25,10 @@ export class FrameCapture {public get isConnectionClosed(): boolean {return this._ws_connection ? this._ws_connection.readyState === this._ws_connection.CLOSED : true;}- public start_frame_capture(onMessageHandler: OnMessageHandler) {- console.log(`FrameCapture: start_frame_capture: ${this._host}`)+ public connect_to_cameras(onMessageHandler: OnMessageHandler) {+ console.log(`WebsocketConnection.connect_to_cameras, connecting to ${this._host_url}`);const decoder = new TextDecoder('utf-8');- this._ws_connection = new WebSocket(this._host);+ this._ws_connection = new WebSocket(this._host_url);this._ws_connection.onmessage = async (ev: MessageEvent<Blob>) => {// Ensure the event.data is a Blob@@ -41,7 +41,7 @@ export class FrameCapture {// Parse the JSON string to a JavaScript objectconst data = JSON.parse(jsonString);- console.log(`Received message with length: ${jsonString.length} from mf_payload# ${data.multi_frame_number}`);+ console.log(`Received message with length: ${jsonString.length} bytes, type: ${typeof data}`);const jpegImagesByCamera = data.jpeg_images;// Iterate over the framesObject to create a data URL for each image@@ -56,14 +56,9 @@ export class FrameCapture {} else {- console.log(`Received message with length: ${JSON.stringify(ev.data)}`);+ console.log(`Received non-Blob message with length: ${JSON.stringify(ev.data)}, type: ${typeof ev.data}`);}};-----}public cleanup() {diff --git a/freemocap-ui/src/views/BoardDetection.tsx b/freemocap-ui/src/views/BoardDetection.tsxindex a5347a94..5967634f 100644--- a/freemocap-ui/src/views/BoardDetection.tsx+++ b/freemocap-ui/src/views/BoardDetection.tsx@@ -1,10 +1,10 @@import React from "react";import {Box} from "@mui/material";-import {CaptureType} from "../services/frame-capture";-import {useFrameCapture} from "../hooks/use-frame-capture";+import {CaptureType} from "../services/websocket-connection";+import {useWebsocket} from "../hooks/use-websocket";export const BoardDetection = () => {- const [frameCapture, data] = useFrameCapture(CaptureType.BoardDetection, 8080);+ const [frameCapture, data] = useWebsocket(CaptureType.BoardDetection, 8080);if (!data) {return null;}diff --git a/freemocap-ui/src/views/BrowserCamerasView.tsx b/freemocap-ui/src/views/BrowserCamerasView.tsxnew file mode 100644index 00000000..d7947633--- /dev/null+++ b/freemocap-ui/src/views/BrowserCamerasView.tsx@@ -0,0 +1,44 @@+import Webcam from "react-webcam";+import {useAsync} from "react-use";+import React from "react";+import {Box, Typography} from "@mui/material";+import {AvailableCameraDevices} from "@/services/detectCameraDevices";++export const BrowserCamerasView = () => {+ const [cams, setCams] = React.useState<MediaDeviceInfo[]>([]);++ useAsync(async () => {+ const foundCams = await new AvailableCameraDevices().findAllCameras();+ setCams(foundCams);+ }, []);++ const supportedConstraints = navigator.mediaDevices.getSupportedConstraints();++ const constraints: MediaStreamConstraints = {+ video: {+ exposureMode: "manual",+ exposureCompensation: "-3"+ } as any+ };++ return (+ <Box display="flex" flexWrap="wrap">+ {cams.map((cam, index) => (+ <Box key={index} m={1} position={"relative"}>+ <Webcam videoConstraints={{...constraints.video as object, deviceId: cam.deviceId}}/>+ <Box position={"absolute"} bottom={0} left={0} bgcolor="rgba(0,0,0,0.5)" color="white" p={1}>+ <Typography variant="body2">{cam.label}</Typography>+ <Typography variant="body2">Device ID: {cam.deviceId}</Typography>+ </Box>+ </Box>+ ))}+ <Box width="100%">+ <pre>+ {`Supported constraints:\n ${JSON.stringify(supportedConstraints, null, 2)}`}+ {`Current Constraints:\n ${JSON.stringify(constraints, null, 2)}`}+ {`Cameras:\n ${JSON.stringify(cams, null, 2)}`}+ </pre>+ </Box>+ </Box>+ );+};diff --git a/freemocap-ui/src/views/Capture.tsx b/freemocap-ui/src/views/Capture.tsxindex 5a98ec09..3e67ec4f 100644--- a/freemocap-ui/src/views/Capture.tsx+++ b/freemocap-ui/src/views/Capture.tsx@@ -3,7 +3,7 @@ import React, {useState} from "react";import Webcam from "react-webcam";import {useAsync} from "react-use";import {StartStopProcess} from "../components/start-stop-process";-import {BrowserCam} from "../services/cam";+import {AvailableCameraDevices} from "../services/detectCameraDevices";import {StreamByDeviceId} from "../services/recorder";class ViewState {@@ -15,7 +15,7 @@ export const WebcamStreamCapture = () => {const [devices, setDevices] = useState<ViewState>(() => new ViewState());// grab all devicesuseAsync(async () => {- const cam = new BrowserCam();+ const cam = new AvailableCameraDevices();const devices = await cam.findAllCameras();setDevices(prev => {return {diff --git a/freemocap-ui/src/views/Config.tsx b/freemocap-ui/src/views/Config.tsxindex 503071ec..e436b833 100644--- a/freemocap-ui/src/views/Config.tsx+++ b/freemocap-ui/src/views/Config.tsx@@ -1,14 +1,14 @@import {Box, Typography} from "@mui/material";import React from "react";import {useAsync} from "react-use";-import {BrowserCam} from "../services/cam";+import {AvailableCameraDevices} from "@/services/detectCameraDevices";export const ConfigView = () => {const [Container] = [Box, Box];const [devices, setDevices] = React.useState<MediaDeviceInfo[]>([]);useAsync(async () => {- const cam = new BrowserCam();- const deviceInfos = await cam.findAllCameras();+ const cam = new AvailableCameraDevices();+ const deviceInfos = await cam.findAllCameras( false);setDevices(deviceInfos)});const devicesWithNames = devices.filter(x => x.label);@@ -23,4 +23,4 @@ export const ConfigView = () => {})}</Container>)-}\ No newline at end of file+}diff --git a/freemocap-ui/src/views/Default.tsx b/freemocap-ui/src/views/Default.tsxindex d8aba341..6b9aa285 100644--- a/freemocap-ui/src/views/Default.tsx+++ b/freemocap-ui/src/views/Default.tsx@@ -1,13 +1,13 @@import {Box, Typography} from "@mui/material";import React from "react";import {useAsync} from "react-use";-import {BrowserCam} from "../services/cam";+import {AvailableCameraDevices} from "../services/detectCameraDevices";export const DefaultView = () => {const [Container] = [Box, Box];const [devices, setDevices] = React.useState<MediaDeviceInfo[]>([]);useAsync(async () => {- const cam = new BrowserCam();+ const cam = new AvailableCameraDevices();const deviceInfos = await cam.findAllCameras();setDevices(deviceInfos)});diff --git a/freemocap-ui/src/views/PythonCamerasView.tsx b/freemocap-ui/src/views/PythonCamerasView.tsxnew file mode 100644index 00000000..20c71b60--- /dev/null+++ b/freemocap-ui/src/views/PythonCamerasView.tsx@@ -0,0 +1,21 @@+import {Box} from "@mui/material";+import {useWebsocket} from "@/hooks/use-websocket";+import {CaptureType} from "@/services/websocket-connection";+import React from "react";++export const PythonCamerasView = () => {++ const [websocketConnection, dataUrls] = useWebsocket(CaptureType.ConnectCameras, 8005);+ if (!dataUrls) {+ return null;+ }+ return (+ <Box sx={{display: "flex", flexDirection: "column"}}>+ Hi wowowwwee+ websocket status: {websocketConnection.isConnectionClosed ? "closed" : "open"}+ {!websocketConnection.isConnectionClosed && Object.entries(dataUrls).map(([cameraId, url]) => (+ <img key={cameraId} src={url} alt={`video capture from camera ${cameraId}`}/>+ ))}+ </Box>+ );+}diff --git a/freemocap-ui/src/views/ShowCameras.tsx b/freemocap-ui/src/views/ShowCameras.tsxdeleted file mode 100644index 06c9e659..00000000--- a/freemocap-ui/src/views/ShowCameras.tsx+++ /dev/null@@ -1,20 +0,0 @@-import {Box} from "@mui/material";-import {useFrameCapture} from "../hooks/use-frame-capture";-import {CaptureType} from "../services/frame-capture";-import React from "react";--export const ShowCameras = () => {-- const [frameCapture, data] = useFrameCapture(CaptureType.ConnectCameras, 8005);- if (!data) {- return null;- }- return (- <Box sx={{display: "flex", flexDirection: "column"}}>- Hi wowowwwee- {!frameCapture.isConnectionClosed && Object.entries(data).map(([cameraId, url]) => (- <img key={cameraId} src={url} alt={`video capture from camera ${cameraId}`}/>- ))}- </Box>- );-}diff --git a/freemocap-ui/src/views/SkeletonDetection.tsx b/freemocap-ui/src/views/SkeletonDetection.tsxindex bce82103..5ffd1e81 100644--- a/freemocap-ui/src/views/SkeletonDetection.tsx+++ b/freemocap-ui/src/views/SkeletonDetection.tsx@@ -1,10 +1,10 @@import React from "react";import {Box} from "@mui/material";-import {CaptureType} from "../services/frame-capture";-import {useFrameCapture} from "../hooks/use-frame-capture";+import {CaptureType} from "../services/websocket-connection";+import {useWebsocket} from "../hooks/use-websocket";export const SkeletonDetection = () => {- const [frameCapture, data] = useFrameCapture(CaptureType.SkeletonDetection, 8080,);+ const [frameCapture, data] = useWebsocket(CaptureType.SkeletonDetection, 8080,);if (!data) {return null;}diff --git a/freemocap-ui/src/views/WebcamJonTest.tsx b/freemocap-ui/src/views/WebcamJonTest.tsxdeleted file mode 100644index 69215af6..00000000--- a/freemocap-ui/src/views/WebcamJonTest.tsx+++ /dev/null@@ -1,42 +0,0 @@-import Webcam from "react-webcam";-import {BrowserCam} from "../services/cam";-import {useAsync} from "react-use";-import React from "react";-import {Box} from "@mui/material";--const cams = new BrowserCam().findAllCameras();--export const WebcamJonTest = () => {- useAsync(async () => {- const cams = await new BrowserCam().findAllCameras();- console.log(cams);- }, []);- const cst = navigator.mediaDevices.getSupportedConstraints()-- const constraints: MediaStreamConstraints = {- video: {- // Jon, you can change stuff here-- // You can select a specific video device by putting a device Id here.- // Check your browser console to take a device you'd like.- // deviceId: "",-- exposureMode: "manual",- exposureCompensation: "-3"- } as any- }--- return (- <Box>- <Webcam videoConstraints={constraints.video}/>- <Box>- <pre>- {`Supported constraints:\n ${JSON.stringify(cst, null, 2)}`}- {`Current Constraints:\n ${JSON.stringify(constraints, null, 2)}`}- {`Cameras:\n ${JSON.stringify(cams, null, 2)}`}- </pre>- </Box>- </Box>- )-}