Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

change the thingy

Endurance Idehen6370b2af6ce5
diff --git a/bin/linux/run_uvicorn_server.sh b/bin/linux/run_uvicorn_server.sh
index d99c1b9b..939a3b18 100755
--- a/bin/linux/run_uvicorn_server.sh
+++ b/bin/linux/run_uvicorn_server.sh
@@ -3,5 +3,6 @@
# Start the webserver
-uvicorn --factory src.api.app_factory:create_app --port 8080 --host 0.0.0.0 --workers 8 --ws websockets
+uvicorn --factory src.api.app_factory:create_app --port 8080 --host 0.0.0.0 --workers 8 --ws websockets --reload
+
diff --git a/freemocap-ui/package.json b/freemocap-ui/package.json
index 98130cc2..1c1bf1ee 100644
--- a/freemocap-ui/package.json
+++ b/freemocap-ui/package.json
@@ -15,10 +15,12 @@
"@mui/system": "^5.2.8",
"@reduxjs/toolkit": "^1.7.1",
"@types/dom-mediacapture-record": "^1.0.11",
+ "@wmik/use-media-recorder": "^1.6.3-beta.0",
"axios": "^0.25.0",
"class-transformer": "0.5.1",
"react": "^17.0.2",
"react-dom": "^17.0.2",
+ "react-media-recorder": "^1.6.6",
"react-number-format": "^4.9.3",
"react-redux": "^7.2.6",
"react-router": "^6.2.1",
diff --git a/freemocap-ui/src/components/webcam/webcam-capture.tsx b/freemocap-ui/src/components/webcam/webcam-capture.tsx
index bd466b5a..6d4a7e46 100644
--- a/freemocap-ui/src/components/webcam/webcam-capture.tsx
+++ b/freemocap-ui/src/components/webcam/webcam-capture.tsx
@@ -1,32 +1,44 @@
-import React from "react";
+import {Box} from "@mui/system";
+import React, {useState} from "react";
import Webcam from "react-webcam";
+import useMediaRecorder from "@wmik/use-media-recorder";
+import {Chunk} from "../../services/Record";
-export const WebcamCapture = () => {
- const [deviceId, setDeviceId] = React.useState<any>({});
- const [devices, setDevices] = React.useState<any>([]);
+interface Props {
+ device: MediaDeviceInfo
+}
- const handleDevices = React.useCallback(mediaDevices =>
- // @ts-ignore
- setDevices(mediaDevices.filter(({ kind }) => kind === "videoinput")),
- [setDevices]
- );
-
- React.useEffect(
- () => {
- navigator.mediaDevices.enumerateDevices().then(handleDevices);
+export const WebcamCapture = (props: Props) => {
+ const {device} = props
+ const [dataAvailable, setDataAvailable] = useState<Chunk[]>([]);
+ const {startRecording, status} = useMediaRecorder({
+ blobOptions: {"type": "video\/mp4"},
+ onError: e => {
+ console.log(e)
},
- [handleDevices]
- );
+ onDataAvailable: (blob) => {
+ const newArray = [...dataAvailable];
+ newArray.push({frameData: blob, timestamp: Date.now()} as Chunk);
+ setDataAvailable(newArray)
+ console.log(newArray)
+ },
+ mediaStreamConstraints: {
+ // video: true
+ video: {
+ deviceId: device.deviceId
+ }
+ }
+ });
- return (
- <>
- {devices.map((device, key) => (
- <div>
- <Webcam audio={false} videoConstraints={{ deviceId: device.deviceId }} />
- {device.label || `Device ${device.deviceId}`}
- </div>
+ if (status === "idle") {
+ console.log('start recording')
+ startRecording(16.67)
+ }
- ))}
- </>
+ return (
+ <Box>
+ {device.label || `Device ${device.deviceId}`}
+ <Webcam audio={false} videoConstraints={{deviceId: device.deviceId}} />
+ </Box>
);
};
\ No newline at end of file
diff --git a/freemocap-ui/src/services/Record.ts b/freemocap-ui/src/services/Record.ts
new file mode 100644
index 00000000..1198214f
--- /dev/null
+++ b/freemocap-ui/src/services/Record.ts
@@ -0,0 +1,19 @@
+export interface Chunk {
+ frameData: Blob,
+ timestamp: number,
+}
+
+export class Record {
+ public static recordStream = (recordedChunks: [Chunk]) => {
+ return (e) => {
+ if (e.data.size > 0) {
+ recordedChunks.push({
+ frameData: e.data,
+ timestamp: Date.now()
+ } as Chunk);
+ console.log(recordedChunks)
+ }
+ }
+ }
+}
+
diff --git a/freemocap-ui/src/services/frame-capture.tsx b/freemocap-ui/src/services/frame-capture.tsx
index a14d8947..2f45d094 100644
--- a/freemocap-ui/src/services/frame-capture.tsx
+++ b/freemocap-ui/src/services/frame-capture.tsx
@@ -8,6 +8,7 @@ export class FrameCapture {
private _ws_connection!: WebSocket;
private readonly _host: string;
private readonly _base_host: string = "ws://localhost:8080/ws";
+ private readonly _revokeQueue: string[] = [];
constructor(
private readonly _webcamId: string,
@@ -16,30 +17,46 @@ export class FrameCapture {
this._host = `${this._base_host}/${this._captureType}/${this._webcamId}`
}
- private _current_data_url!: string;
-
- public get current_data_url(): string {
- return this._current_data_url;
- }
-
public get isConnectionClosed(): boolean {
return this._ws_connection.readyState === this._ws_connection.CLOSED;
}
public start_frame_capture(onMessageHandler: OnMessageHandler) {
this._ws_connection = new WebSocket(this._host);
+
+ this._ws_connection.onopen = async () => {
+ this._check_and_revoke();
+ }
+
this._ws_connection.onmessage = async (ev: MessageEvent<Blob>) => {
- if (this._current_data_url) {
- URL.revokeObjectURL(this._current_data_url);
- }
const new_data_url = URL.createObjectURL(ev.data);
- this._current_data_url = new_data_url;
await onMessageHandler(ev, new_data_url);
+ this._revokeQueue.push(new_data_url);
+ }
+
+ this._ws_connection.onclose = async () => {
+ this._full_revoke();
}
}
public cleanup() {
this._ws_connection.close();
}
+
+ private _check_and_revoke() {
+ if (this._revokeQueue.length > 100) {
+ const first50Images = this._revokeQueue.splice(0, 50);
+ first50Images.forEach((imgUrl) => {
+ URL.revokeObjectURL(imgUrl);
+ })
+ }
+ }
+
+ private _full_revoke() {
+ this._revokeQueue.forEach((imgUrl) => {
+ URL.revokeObjectURL(imgUrl);
+ });
+ }
+
}
diff --git a/freemocap-ui/src/views/prod/SetupAndPreviewCamera.tsx b/freemocap-ui/src/views/prod/SetupAndPreviewCamera.tsx
index c3f3d517..a6a05788 100644
--- a/freemocap-ui/src/views/prod/SetupAndPreviewCamera.tsx
+++ b/freemocap-ui/src/views/prod/SetupAndPreviewCamera.tsx
@@ -20,7 +20,7 @@ export const SetupAndPreviewCamera = (props: Props) => {
<Box>
<Box display={'flex'} flexDirection={'column'} width={400}>
<Button>Preview in CV2</Button>
- {!frameCapture.isConnectionClosed && <img src={frameCapture.current_data_url} alt={"video capture"} />}
+ {!frameCapture.isConnectionClosed && <img src={data} alt={"video capture"} />}
</Box>
<ConfigForm onSubmit={onSubmit}/>
</Box>
diff --git a/freemocap-ui/src/views/prod/SetupAndPreviewView.tsx b/freemocap-ui/src/views/prod/SetupAndPreviewView.tsx
index bca5804a..0b3c26bf 100644
--- a/freemocap-ui/src/views/prod/SetupAndPreviewView.tsx
+++ b/freemocap-ui/src/views/prod/SetupAndPreviewView.tsx
@@ -2,25 +2,30 @@ import {Box} from "@mui/material";
import React, {useState} from "react";
import {useAsync} from "react-use";
import axios from "axios";
-import {SessionWizard} from "../sessionWizard/SessionWizard";
+import {WebcamCapture} from "../../components/webcam/webcam-capture";
-export const SetupAndPreviewView = () => {
+export const usePythonCameraDetection = () => {
const [webcamIds, setWebcamIds] = useState();
-
- useAsync(async () => {
+ const response = useAsync(async () => {
const response = await axios.get('http://localhost:8080/camera/detect');
const webcamIds = response.data.cameras_found_list.map(x => x.webcam_id);
setWebcamIds(webcamIds)
}, [])
+ return [webcamIds, response];
+}
- if (!webcamIds) {
- // Small loading indicator
- return null;
- }
-
+export const SetupAndPreviewView = () => {
+ const [devices, setDevices] = useState<MediaDeviceInfo[]>();
+ useAsync(async () => {
+ const devices = await navigator.mediaDevices.enumerateDevices()
+ const videoDevices = devices.filter(({ kind }) => kind.toLowerCase() === "videoinput")
+ setDevices(videoDevices)
+ }, [])
return (
<Box>
- <SessionWizard webcamIds={webcamIds} />
+ {devices?.map(dev => {
+ return <WebcamCapture device={dev} />
+ })}
</Box>
)
}
\ No newline at end of file
diff --git a/freemocap-ui/src/views/sessionWizard/SessionWizard.tsx b/freemocap-ui/src/views/sessionWizard/SessionWizard.tsx
index 32d78314..da902e6e 100644
--- a/freemocap-ui/src/views/sessionWizard/SessionWizard.tsx
+++ b/freemocap-ui/src/views/sessionWizard/SessionWizard.tsx
@@ -3,6 +3,8 @@ import {Button, Typography} from "@mui/material";
import {Box} from "@mui/system";
import {useWizard, Wizard} from "react-use-wizard";
import axios from "axios";
+import {useAsync} from "react-use";
+import {useParams} from "react-router";
interface Props {
webcamIds: string[];
@@ -11,13 +13,13 @@ interface Props {
export const SessionWizard = (props: Props) => {
// generate steps within the step wizard
const {webcamIds} = props
-
+ const { sessionId } = useParams();
return (
<Wizard>
{webcamIds.map(webcamId => {
return <CamStep webcamId={webcamId} />
})}
- {/*<IMShowPreviewCamera />*/}
+ <IMShowPreviewCamera sessionId={sessionId}/>
</Wizard>
)
}
@@ -38,22 +40,25 @@ export const CamStep = (props: CamStepProps) => {
</Box>
)
}
-//
-// interface PreviewCamera {}
-// export const IMShowPreviewCamera = () => {
-// const {handleStep, previousStep, nextStep, stepCount, isLastStep} = useWizard();
-// useAsync(async () => {
-// await axios.post('http://localhost:8080/camera/cv2_imshow_one_camera', {
-// session_id
-// });
-// }, []);
-// return (
-// <Box>
-// <Typography>External Windows should pop up showing the images from your webcams all together.</Typography>
-// <Typography>If you are ready to record, and the configuration seems correct, please continue.</Typography>
-// <Typography>If not, please go to the previous steps and change the configuration</Typography>
-// <Button onClick={previousStep}>No, I want to go back</Button>
-// <Button onClick={nextStep}>Yes, Im ready to record</Button>
-// </Box>
-// )
-// }
+
+interface PreviewCameraProps {
+ sessionId: string | undefined
+}
+
+export const IMShowPreviewCamera = (props: PreviewCameraProps) => {
+ const {handleStep, previousStep, nextStep, stepCount, isLastStep} = useWizard();
+ useAsync(async () => {
+ await axios.post('http://localhost:8080/camera/show_cameras', {
+ session_id: props.sessionId
+ });
+ }, []);
+ return (
+ <Box>
+ <Typography>External Windows should pop up showing the images from your webcams all together.</Typography>
+ <Typography>If you are ready to record, and the configuration seems correct, please continue.</Typography>
+ <Typography>If not, please go to the previous steps and change the configuration</Typography>
+ <Button onClick={previousStep}>No, I want to go back</Button>
+ <Button onClick={nextStep}>Yes, Im ready to calibrate</Button>
+ </Box>
+ )
+}
diff --git a/src/api/routes/camera/camera_route.py b/src/api/routes/camera/camera_route.py
index 263befd1..81db2eb6 100644
--- a/src/api/routes/camera/camera_route.py
+++ b/src/api/routes/camera/camera_route.py
@@ -2,6 +2,7 @@ import logging
import traceback
import cv2
+import pyglview as pyglview
from fastapi import APIRouter
from pydantic import BaseModel
@@ -17,14 +18,15 @@ logger = logging.getLogger(__name__)
class CameraPreviewModel(BaseModel):
session_id: str = None
+
@camera_router.post("/camera/config/{session_id}")
async def config_cam(webcam_config_model: WebcamConfigModel, session_id):
s = UserConfigService()
return s.save_webcam_config_to_disk(webcam_config_model, session_id)
-@camera_router.post("/camera/cv2_imshow_one_camera/{webcam_id}")
-async def cv2_imshow_one_camera(webcam_id: str, camera_preview_model: CameraPreviewModel = CameraPreviewModel()):
+@camera_router.post("/camera/show_camera/{webcam_id}")
+async def show_camera(webcam_id: str, camera_preview_model: CameraPreviewModel = CameraPreviewModel()):
cv_cam_manager = OpenCVCameraManager(session_id=camera_preview_model.session_id)
with cv_cam_manager.start_capture_session_single_cam(webcam_id) as connected_camera_and_writer:
should_continue = True
@@ -42,7 +44,7 @@ async def cv2_imshow_one_camera(webcam_id: str, camera_preview_model: CameraPrev
cv2.destroyWindow(cv2_window_name)
-@camera_router.post("/camera/cv2_imshow_all_cameras")
+@camera_router.post("/camera/show_cameras")
async def cv2_imshow_all_camera(camera_preview_model: CameraPreviewModel = CameraPreviewModel()):
cv_cam_manager = OpenCVCameraManager(session_id=camera_preview_model.session_id)
with cv_cam_manager.start_capture_session_all_cams() as connected_cameras_dict:
@@ -50,16 +52,20 @@ async def cv2_imshow_all_camera(camera_preview_model: CameraPreviewModel = Camer
should_continue = True
while should_continue:
for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():
+ exit_key = cv2.waitKey(1)
+ if exit_key == 27:
+ logger.info("ESC has been pressed.")
+ should_continue = False
+ cv2.destroyWindow(cv2_window_name)
if not this_open_cv_camera.new_frame_ready:
continue
- cv2.imshow('(ESC to close) PREVIEWING camera_' + this_webcam_id, this_open_cv_camera.latest_frame.image)
- # exit loop when user presses ESC key
- exit_key = cv2.waitKey(1)
- if exit_key == 27:
- logger.info("ESC has been pressed.")
- should_continue = False
- cv2.destroyAllWindows() #TODO - close each named window specificially by name
-
+ cv2_window_name = '(ESC to close) PREVIEWING camera_' + this_webcam_id
+ cv2.imshow(cv2_window_name, this_open_cv_camera.latest_frame.image)
+ exit_key = cv2.waitKey(1)
+ if exit_key == 27:
+ logger.info("ESC has been pressed.")
+ should_continue = False
+ cv2.destroyWindow(cv2_window_name)
@camera_router.get("/camera/detect")
async def get_cameras():
diff --git a/src/api/routes/session/session_router.py b/src/api/routes/session/session_router.py
index 0326828c..f3886858 100644
--- a/src/api/routes/session/session_router.py
+++ b/src/api/routes/session/session_router.py
@@ -5,7 +5,7 @@ from typing import Optional
from fastapi import APIRouter
from pydantic import BaseModel
-from src.config.home_dir import os_independent_home_dir, create_session_id, get_session_folder_path
+from src.config.home_dir import create_session_id, get_session_folder_path
logger = logging.getLogger(__name__)
@@ -26,10 +26,11 @@ async def create_session(session_create_model: SessionCreateModel):
session_id = create_session_id(session_create_model.user_session_tag_str)
session_path = Path(get_session_folder_path(session_id))
logger.info(f'Creating session folder at: {str(session_path)}')
- session_path.mkdir(parents=True, exist_ok=False)
+ session_path.mkdir(parents=True, exist_ok=True)
return SessionResponse(session_id=session_id,
session_path=str(session_path))
+
@session_router.post("session/calibrate")
async def calibrate_session():
"""launch camera calibration (or load previous calibration)"""
diff --git a/src/config/home_dir.py b/src/config/home_dir.py
index ff1e2bad..06d0b944 100644
--- a/src/config/home_dir.py
+++ b/src/config/home_dir.py
@@ -20,7 +20,6 @@ def get_freemocap_data_folder_path():
freemocap_data_folder_path = Path(os_independent_home_dir(), BASE_FOLDER_NAME)
freemocap_data_folder_path.mkdir(exist_ok=True,parents=True)
return str(freemocap_data_folder_path)
- return str(freemocap_data_folder_path)
def get_session_folder_path(session_id: str):