Free Motion Capture for Everyone πβ¨
TypeScript
change the thingy
Endurance Idehen6370b2af6ce5
diff --git a/bin/linux/run_uvicorn_server.sh b/bin/linux/run_uvicorn_server.shindex d99c1b9b..939a3b18 100755--- a/bin/linux/run_uvicorn_server.sh+++ b/bin/linux/run_uvicorn_server.sh@@ -3,5 +3,6 @@# Start the webserver-uvicorn --factory src.api.app_factory:create_app --port 8080 --host 0.0.0.0 --workers 8 --ws websockets+uvicorn --factory src.api.app_factory:create_app --port 8080 --host 0.0.0.0 --workers 8 --ws websockets --reload+diff --git a/freemocap-ui/package.json b/freemocap-ui/package.jsonindex 98130cc2..1c1bf1ee 100644--- a/freemocap-ui/package.json+++ b/freemocap-ui/package.json@@ -15,10 +15,12 @@"@mui/system": "^5.2.8","@reduxjs/toolkit": "^1.7.1","@types/dom-mediacapture-record": "^1.0.11",+ "@wmik/use-media-recorder": "^1.6.3-beta.0","axios": "^0.25.0","class-transformer": "0.5.1","react": "^17.0.2","react-dom": "^17.0.2",+ "react-media-recorder": "^1.6.6","react-number-format": "^4.9.3","react-redux": "^7.2.6","react-router": "^6.2.1",diff --git a/freemocap-ui/src/components/webcam/webcam-capture.tsx b/freemocap-ui/src/components/webcam/webcam-capture.tsxindex bd466b5a..6d4a7e46 100644--- a/freemocap-ui/src/components/webcam/webcam-capture.tsx+++ b/freemocap-ui/src/components/webcam/webcam-capture.tsx@@ -1,32 +1,44 @@-import React from "react";+import {Box} from "@mui/system";+import React, {useState} from "react";import Webcam from "react-webcam";+import useMediaRecorder from "@wmik/use-media-recorder";+import {Chunk} from "../../services/Record";-export const WebcamCapture = () => {- const [deviceId, setDeviceId] = React.useState<any>({});- const [devices, setDevices] = React.useState<any>([]);+interface Props {+ device: MediaDeviceInfo+}- const handleDevices = React.useCallback(mediaDevices =>- // @ts-ignore- setDevices(mediaDevices.filter(({ kind }) => kind === "videoinput")),- [setDevices]- );-- React.useEffect(- () => {- navigator.mediaDevices.enumerateDevices().then(handleDevices);+export const WebcamCapture = (props: Props) => {+ const {device} = props+ const [dataAvailable, setDataAvailable] = useState<Chunk[]>([]);+ const {startRecording, status} = useMediaRecorder({+ blobOptions: {"type": "video\/mp4"},+ onError: e => {+ console.log(e)},- [handleDevices]- );+ onDataAvailable: (blob) => {+ const newArray = [...dataAvailable];+ newArray.push({frameData: blob, timestamp: Date.now()} as Chunk);+ setDataAvailable(newArray)+ console.log(newArray)+ },+ mediaStreamConstraints: {+ // video: true+ video: {+ deviceId: device.deviceId+ }+ }+ });- return (- <>- {devices.map((device, key) => (- <div>- <Webcam audio={false} videoConstraints={{ deviceId: device.deviceId }} />- {device.label || `Device ${device.deviceId}`}- </div>+ if (status === "idle") {+ console.log('start recording')+ startRecording(16.67)+ }- ))}- </>+ return (+ <Box>+ {device.label || `Device ${device.deviceId}`}+ <Webcam audio={false} videoConstraints={{deviceId: device.deviceId}} />+ </Box>);};\ No newline at end of filediff --git a/freemocap-ui/src/services/Record.ts b/freemocap-ui/src/services/Record.tsnew file mode 100644index 00000000..1198214f--- /dev/null+++ b/freemocap-ui/src/services/Record.ts@@ -0,0 +1,19 @@+export interface Chunk {+ frameData: Blob,+ timestamp: number,+}++export class Record {+ public static recordStream = (recordedChunks: [Chunk]) => {+ return (e) => {+ if (e.data.size > 0) {+ recordedChunks.push({+ frameData: e.data,+ timestamp: Date.now()+ } as Chunk);+ console.log(recordedChunks)+ }+ }+ }+}+diff --git a/freemocap-ui/src/services/frame-capture.tsx b/freemocap-ui/src/services/frame-capture.tsxindex a14d8947..2f45d094 100644--- a/freemocap-ui/src/services/frame-capture.tsx+++ b/freemocap-ui/src/services/frame-capture.tsx@@ -8,6 +8,7 @@ export class FrameCapture {private _ws_connection!: WebSocket;private readonly _host: string;private readonly _base_host: string = "ws://localhost:8080/ws";+ private readonly _revokeQueue: string[] = [];constructor(private readonly _webcamId: string,@@ -16,30 +17,46 @@ export class FrameCapture {this._host = `${this._base_host}/${this._captureType}/${this._webcamId}`}- private _current_data_url!: string;-- public get current_data_url(): string {- return this._current_data_url;- }-public get isConnectionClosed(): boolean {return this._ws_connection.readyState === this._ws_connection.CLOSED;}public start_frame_capture(onMessageHandler: OnMessageHandler) {this._ws_connection = new WebSocket(this._host);++ this._ws_connection.onopen = async () => {+ this._check_and_revoke();+ }+this._ws_connection.onmessage = async (ev: MessageEvent<Blob>) => {- if (this._current_data_url) {- URL.revokeObjectURL(this._current_data_url);- }const new_data_url = URL.createObjectURL(ev.data);- this._current_data_url = new_data_url;await onMessageHandler(ev, new_data_url);+ this._revokeQueue.push(new_data_url);+ }++ this._ws_connection.onclose = async () => {+ this._full_revoke();}}public cleanup() {this._ws_connection.close();}++ private _check_and_revoke() {+ if (this._revokeQueue.length > 100) {+ const first50Images = this._revokeQueue.splice(0, 50);+ first50Images.forEach((imgUrl) => {+ URL.revokeObjectURL(imgUrl);+ })+ }+ }++ private _full_revoke() {+ this._revokeQueue.forEach((imgUrl) => {+ URL.revokeObjectURL(imgUrl);+ });+ }+}diff --git a/freemocap-ui/src/views/prod/SetupAndPreviewCamera.tsx b/freemocap-ui/src/views/prod/SetupAndPreviewCamera.tsxindex c3f3d517..a6a05788 100644--- a/freemocap-ui/src/views/prod/SetupAndPreviewCamera.tsx+++ b/freemocap-ui/src/views/prod/SetupAndPreviewCamera.tsx@@ -20,7 +20,7 @@ export const SetupAndPreviewCamera = (props: Props) => {<Box><Box display={'flex'} flexDirection={'column'} width={400}><Button>Preview in CV2</Button>- {!frameCapture.isConnectionClosed && <img src={frameCapture.current_data_url} alt={"video capture"} />}+ {!frameCapture.isConnectionClosed && <img src={data} alt={"video capture"} />}</Box><ConfigForm onSubmit={onSubmit}/></Box>diff --git a/freemocap-ui/src/views/prod/SetupAndPreviewView.tsx b/freemocap-ui/src/views/prod/SetupAndPreviewView.tsxindex bca5804a..0b3c26bf 100644--- a/freemocap-ui/src/views/prod/SetupAndPreviewView.tsx+++ b/freemocap-ui/src/views/prod/SetupAndPreviewView.tsx@@ -2,25 +2,30 @@ import {Box} from "@mui/material";import React, {useState} from "react";import {useAsync} from "react-use";import axios from "axios";-import {SessionWizard} from "../sessionWizard/SessionWizard";+import {WebcamCapture} from "../../components/webcam/webcam-capture";-export const SetupAndPreviewView = () => {+export const usePythonCameraDetection = () => {const [webcamIds, setWebcamIds] = useState();-- useAsync(async () => {+ const response = useAsync(async () => {const response = await axios.get('http://localhost:8080/camera/detect');const webcamIds = response.data.cameras_found_list.map(x => x.webcam_id);setWebcamIds(webcamIds)}, [])+ return [webcamIds, response];+}- if (!webcamIds) {- // Small loading indicator- return null;- }-+export const SetupAndPreviewView = () => {+ const [devices, setDevices] = useState<MediaDeviceInfo[]>();+ useAsync(async () => {+ const devices = await navigator.mediaDevices.enumerateDevices()+ const videoDevices = devices.filter(({ kind }) => kind.toLowerCase() === "videoinput")+ setDevices(videoDevices)+ }, [])return (<Box>- <SessionWizard webcamIds={webcamIds} />+ {devices?.map(dev => {+ return <WebcamCapture device={dev} />+ })}</Box>)}\ No newline at end of filediff --git a/freemocap-ui/src/views/sessionWizard/SessionWizard.tsx b/freemocap-ui/src/views/sessionWizard/SessionWizard.tsxindex 32d78314..da902e6e 100644--- a/freemocap-ui/src/views/sessionWizard/SessionWizard.tsx+++ b/freemocap-ui/src/views/sessionWizard/SessionWizard.tsx@@ -3,6 +3,8 @@ import {Button, Typography} from "@mui/material";import {Box} from "@mui/system";import {useWizard, Wizard} from "react-use-wizard";import axios from "axios";+import {useAsync} from "react-use";+import {useParams} from "react-router";interface Props {webcamIds: string[];@@ -11,13 +13,13 @@ interface Props {export const SessionWizard = (props: Props) => {// generate steps within the step wizardconst {webcamIds} = props-+ const { sessionId } = useParams();return (<Wizard>{webcamIds.map(webcamId => {return <CamStep webcamId={webcamId} />})}- {/*<IMShowPreviewCamera />*/}+ <IMShowPreviewCamera sessionId={sessionId}/></Wizard>)}@@ -38,22 +40,25 @@ export const CamStep = (props: CamStepProps) => {</Box>)}-//-// interface PreviewCamera {}-// export const IMShowPreviewCamera = () => {-// const {handleStep, previousStep, nextStep, stepCount, isLastStep} = useWizard();-// useAsync(async () => {-// await axios.post('http://localhost:8080/camera/cv2_imshow_one_camera', {-// session_id-// });-// }, []);-// return (-// <Box>-// <Typography>External Windows should pop up showing the images from your webcams all together.</Typography>-// <Typography>If you are ready to record, and the configuration seems correct, please continue.</Typography>-// <Typography>If not, please go to the previous steps and change the configuration</Typography>-// <Button onClick={previousStep}>No, I want to go back</Button>-// <Button onClick={nextStep}>Yes, Im ready to record</Button>-// </Box>-// )-// }++interface PreviewCameraProps {+ sessionId: string | undefined+}++export const IMShowPreviewCamera = (props: PreviewCameraProps) => {+ const {handleStep, previousStep, nextStep, stepCount, isLastStep} = useWizard();+ useAsync(async () => {+ await axios.post('http://localhost:8080/camera/show_cameras', {+ session_id: props.sessionId+ });+ }, []);+ return (+ <Box>+ <Typography>External Windows should pop up showing the images from your webcams all together.</Typography>+ <Typography>If you are ready to record, and the configuration seems correct, please continue.</Typography>+ <Typography>If not, please go to the previous steps and change the configuration</Typography>+ <Button onClick={previousStep}>No, I want to go back</Button>+ <Button onClick={nextStep}>Yes, Im ready to calibrate</Button>+ </Box>+ )+}diff --git a/src/api/routes/camera/camera_route.py b/src/api/routes/camera/camera_route.pyindex 263befd1..81db2eb6 100644--- a/src/api/routes/camera/camera_route.py+++ b/src/api/routes/camera/camera_route.py@@ -2,6 +2,7 @@ import loggingimport tracebackimport cv2+import pyglview as pyglviewfrom fastapi import APIRouterfrom pydantic import BaseModel@@ -17,14 +18,15 @@ logger = logging.getLogger(__name__)class CameraPreviewModel(BaseModel):session_id: str = None+@camera_router.post("/camera/config/{session_id}")async def config_cam(webcam_config_model: WebcamConfigModel, session_id):s = UserConfigService()return s.save_webcam_config_to_disk(webcam_config_model, session_id)-@camera_router.post("/camera/cv2_imshow_one_camera/{webcam_id}")-async def cv2_imshow_one_camera(webcam_id: str, camera_preview_model: CameraPreviewModel = CameraPreviewModel()):+@camera_router.post("/camera/show_camera/{webcam_id}")+async def show_camera(webcam_id: str, camera_preview_model: CameraPreviewModel = CameraPreviewModel()):cv_cam_manager = OpenCVCameraManager(session_id=camera_preview_model.session_id)with cv_cam_manager.start_capture_session_single_cam(webcam_id) as connected_camera_and_writer:should_continue = True@@ -42,7 +44,7 @@ async def cv2_imshow_one_camera(webcam_id: str, camera_preview_model: CameraPrevcv2.destroyWindow(cv2_window_name)-@camera_router.post("/camera/cv2_imshow_all_cameras")+@camera_router.post("/camera/show_cameras")async def cv2_imshow_all_camera(camera_preview_model: CameraPreviewModel = CameraPreviewModel()):cv_cam_manager = OpenCVCameraManager(session_id=camera_preview_model.session_id)with cv_cam_manager.start_capture_session_all_cams() as connected_cameras_dict:@@ -50,16 +52,20 @@ async def cv2_imshow_all_camera(camera_preview_model: CameraPreviewModel = Camershould_continue = Truewhile should_continue:for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():+ exit_key = cv2.waitKey(1)+ if exit_key == 27:+ logger.info("ESC has been pressed.")+ should_continue = False+ cv2.destroyWindow(cv2_window_name)if not this_open_cv_camera.new_frame_ready:continue- cv2.imshow('(ESC to close) PREVIEWING camera_' + this_webcam_id, this_open_cv_camera.latest_frame.image)- # exit loop when user presses ESC key- exit_key = cv2.waitKey(1)- if exit_key == 27:- logger.info("ESC has been pressed.")- should_continue = False- cv2.destroyAllWindows() #TODO - close each named window specificially by name-+ cv2_window_name = '(ESC to close) PREVIEWING camera_' + this_webcam_id+ cv2.imshow(cv2_window_name, this_open_cv_camera.latest_frame.image)+ exit_key = cv2.waitKey(1)+ if exit_key == 27:+ logger.info("ESC has been pressed.")+ should_continue = False+ cv2.destroyWindow(cv2_window_name)@camera_router.get("/camera/detect")async def get_cameras():diff --git a/src/api/routes/session/session_router.py b/src/api/routes/session/session_router.pyindex 0326828c..f3886858 100644--- a/src/api/routes/session/session_router.py+++ b/src/api/routes/session/session_router.py@@ -5,7 +5,7 @@ from typing import Optionalfrom fastapi import APIRouterfrom pydantic import BaseModel-from src.config.home_dir import os_independent_home_dir, create_session_id, get_session_folder_path+from src.config.home_dir import create_session_id, get_session_folder_pathlogger = logging.getLogger(__name__)@@ -26,10 +26,11 @@ async def create_session(session_create_model: SessionCreateModel):session_id = create_session_id(session_create_model.user_session_tag_str)session_path = Path(get_session_folder_path(session_id))logger.info(f'Creating session folder at: {str(session_path)}')- session_path.mkdir(parents=True, exist_ok=False)+ session_path.mkdir(parents=True, exist_ok=True)return SessionResponse(session_id=session_id,session_path=str(session_path))+@session_router.post("session/calibrate")async def calibrate_session():"""launch camera calibration (or load previous calibration)"""diff --git a/src/config/home_dir.py b/src/config/home_dir.pyindex ff1e2bad..06d0b944 100644--- a/src/config/home_dir.py+++ b/src/config/home_dir.py@@ -20,7 +20,6 @@ def get_freemocap_data_folder_path():freemocap_data_folder_path = Path(os_independent_home_dir(), BASE_FOLDER_NAME)freemocap_data_folder_path.mkdir(exist_ok=True,parents=True)return str(freemocap_data_folder_path)- return str(freemocap_data_folder_path)def get_session_folder_path(session_id: str):