Free Motion Capture for Everyone πβ¨
TypeScript
OpenCVCamera comes with a viewer now with a "show" function Cv Cam has a multiprocess implementation in our experiments
Endurance Idehenca9167cfd8df
diff --git a/freemocap-ui/config-overrides.js b/freemocap-ui/config-overrides.jsnew file mode 100644index 00000000..00d9dc67--- /dev/null+++ b/freemocap-ui/config-overrides.js@@ -0,0 +1,23 @@+const webpack = require('webpack');+module.exports = function override(config) {+ const fallback = config.resolve.fallback || {};+ Object.assign(fallback, {+ "crypto": require.resolve("crypto-browserify"),+ "stream": require.resolve("stream-browserify"),+ "assert": require.resolve("assert"),+ "http": require.resolve("stream-http"),+ "https": require.resolve("https-browserify"),+ "os": require.resolve("os-browserify"),+ "url": require.resolve("url"),+ fs: false,+ path: false,+ })+ config.resolve.fallback = fallback;+ config.plugins = (config.plugins || []).concat([+ new webpack.ProvidePlugin({+ process: 'process/browser',+ Buffer: ['buffer', 'Buffer']+ })+ ])+ return config;+}\ No newline at end of filediff --git a/freemocap-ui/package.json b/freemocap-ui/package.jsonindex 1c1bf1ee..4e6c034e 100644--- a/freemocap-ui/package.json+++ b/freemocap-ui/package.json@@ -14,7 +14,10 @@"@mui/styled-engine": "^5.2.6","@mui/system": "^5.2.8","@reduxjs/toolkit": "^1.7.1",+ "@shopify/react-web-worker": "^5.0.1",+ "@techstark/opencv-js": "^4.5.5-release.2","@types/dom-mediacapture-record": "^1.0.11",+ "@types/w3c-image-capture": "^1.0.6","@wmik/use-media-recorder": "^1.6.3-beta.0","axios": "^0.25.0","class-transformer": "0.5.1",@@ -28,7 +31,7 @@"react-scripts": "5.0.0","react-step-wizard": "^5.3.11","react-use": "^17.3.2",- "react-use-websocket": "^2.9.1",+ "react-use-websocket": "^3.0.0","react-use-wizard": "^2.2.0","react-webcam": "^6.0.0","reflect-metadata": "^0.1.13",@@ -47,14 +50,24 @@"@types/react": "^17.0.38","@types/react-dom": "^17.0.11","@types/react-redux": "^7.1.11",- "@types/react-router-dom": "^5.1.6"+ "@types/react-router-dom": "^5.1.6",+ "assert": "^2.0.0",+ "buffer": "^6.0.3",+ "crypto-browserify": "^3.12.0",+ "https-browserify": "^1.0.0",+ "os-browserify": "^0.3.0",+ "process": "^0.11.10",+ "react-app-rewired": "^2.2.1",+ "stream-browserify": "^3.0.0",+ "stream-http": "^3.2.0",+ "url": "^0.11.0"},"scripts": {- "start": "react-scripts start","start:https": "HTTPS=true SSL_CRT_FILE=./localhost.pem SSL_KEY_FILE=./localhost-key.pem react-scripts start","certs": "./scripts/cert.sh",- "build": "react-scripts build",- "test": "react-scripts test",+ "start": "react-app-rewired start",+ "build": "react-app-rewired build",+ "test": "react-app-rewired test","eject": "react-scripts eject"},"eslintConfig": {diff --git a/freemocap-ui/public/index.html b/freemocap-ui/public/index.htmlindex 1dbdca65..b8c495c0 100644--- a/freemocap-ui/public/index.html+++ b/freemocap-ui/public/index.html@@ -40,4 +40,7 @@To create a production bundle, use `npm run build` or `yarn build`.--></body>+ <script type="text/javascript" src="https://unpkg.com/[email protected]/lib/opencv.js"></script>+<!-- <script src="https://unpkg.com/opencv-javascript"></script>-->+</html>diff --git a/freemocap-ui/src/components/webcam/LivePreview.tsx b/freemocap-ui/src/components/webcam/LivePreview.tsxnew file mode 100644index 00000000..7e13d160--- /dev/null+++ b/freemocap-ui/src/components/webcam/LivePreview.tsx@@ -0,0 +1,23 @@+import React from "react";++interface Props {+ stream: MediaStream+}++export function LiveStreamPreview({ stream }) {+ let videoPreviewRef = React.useRef<HTMLVideoElement>();++ React.useEffect(() => {+ if (videoPreviewRef.current && stream) {+ // @ts-ignore+ videoPreviewRef.current.srcObject = stream;+ }+ }, [stream]);++ if (!stream) {+ return null;+ }++ // @ts-ignore+ return <video ref={videoPreviewRef} width={800} height={600} autoPlay />;+}diff --git a/freemocap-ui/src/components/webcam/webcam-capture.tsx b/freemocap-ui/src/components/webcam/webcam-capture.tsxindex 696e313f..95217baf 100644--- a/freemocap-ui/src/components/webcam/webcam-capture.tsx+++ b/freemocap-ui/src/components/webcam/webcam-capture.tsx@@ -1,8 +1,7 @@-import {Box} from "@mui/system";-import React, {useState} from "react";-import Webcam from "react-webcam";-import useMediaRecorder from "@wmik/use-media-recorder";-import {Chunk} from "../../services/Record";+import React, {useEffect, useState} from "react";+import {useDeviceStream} from "../../hooks/use-device-stream";+import {Capture} from "../../services/Capture";+import {Button} from "@mui/material";interface Props {device: MediaDeviceInfo@@ -10,35 +9,30 @@ interface Props {export const WebcamCapture = (props: Props) => {const {device} = props- const [dataAvailable, setDataAvailable] = useState<Chunk[]>([]);- const {startRecording, status} = useMediaRecorder({- onDataAvailable: (blob) => {- const newArray = [...dataAvailable];- newArray.push({frameData: blob, timestamp: Date.now()} as Chunk);- setDataAvailable(newArray)- console.log(newArray)- },- // blobOptions: {"type": "video\/mp4"},- // onError: e => {- // console.log(e)- // },- mediaStreamConstraints: {- // video: true- video: {- deviceId: device.deviceId- }+ const videoRef = React.useRef<HTMLVideoElement>(null);+ const stream = useDeviceStream(device)+ // const { sendMessage, getWebSocket } = useWebSocket("ws://localhost:8080/ws/hello_world")+ const [capture, setCapture] = useState<Capture>()+ useEffect(() => {+ const {current} = videoRef+ if (current) {+ // @ts-ignore+ current.srcObject = stream+ const capture = new Capture(videoRef)+ setCapture(capture)}- });+ }, [stream, videoRef])- if (status === "idle") {- console.log('start recording')- startRecording(16.67)+ if (!stream) {+ return null}-return (- <Box>- {device.label || `Device ${device.deviceId}`}- <Webcam audio={true} videoConstraints={{deviceId: device.deviceId}} />- </Box>+ <>+ <video ref={videoRef} height={600} width={800} autoPlay />+ <canvas id={"canvasOutput"} />+ <Button onClick={() => {+ capture?.processVideo()+ }}>Start Capture</Button>+ </>);};\ No newline at end of filediff --git a/freemocap-ui/src/hooks/use-device-stream.ts b/freemocap-ui/src/hooks/use-device-stream.tsnew file mode 100644index 00000000..05cd766b--- /dev/null+++ b/freemocap-ui/src/hooks/use-device-stream.ts@@ -0,0 +1,12 @@+import {useAsync} from "react-use";++export const useDeviceStream = (device: MediaDeviceInfo) => {+ const {value} = useAsync(async () => {+ return await navigator.mediaDevices.getUserMedia({+ video: {+ deviceId: device.deviceId+ },+ })+ }, [device])+ return value+}\ No newline at end of filediff --git a/freemocap-ui/src/services/Capture.tsx b/freemocap-ui/src/services/Capture.tsxnew file mode 100644index 00000000..25332248--- /dev/null+++ b/freemocap-ui/src/services/Capture.tsx@@ -0,0 +1,24 @@+import cv from "@techstark/opencv-js";++export class Capture {+ private FPS = 60;+ private _cap: cv.VideoCapture;+ private _ws: WebSocket;++ constructor(videoElement: any) {+ const {height, width, current} = videoElement+ console.log(height, width, current)+ this._cap = new cv.VideoCapture(current);+ this._ws = new WebSocket("ws://localhost:8080/ws/hello_world")+ }++ processVideo = () => {+ setInterval(() => {+ const image = new cv.Mat(600, 800, cv.CV_8UC4)+ this._cap.read(image);+ // cv.imshow('canvasOutput', image);+ this._ws.send(image.data)+ image.delete()+ }, 10)+ }+}\ No newline at end of filediff --git a/freemocap-ui/src/services/image_send_worker.ts b/freemocap-ui/src/services/image_send_worker.tsnew file mode 100644index 00000000..d70f20ae--- /dev/null+++ b/freemocap-ui/src/services/image_send_worker.ts@@ -0,0 +1,16 @@+const webSocket = new WebSocket("ws://localhost:8080/ws/hello_world")++export const sendImageByRef = (camRef) => {+ const imageSrc = camRef.current.getScreenshot({+ height: 1080,+ width: 1920,+ });+ webSocket.send(imageSrc)+}++export const sendImageByDevice = async (device: MediaDeviceInfo) => {+ // const stream = await navigator.mediaDevices.getUserMedia({ video: { deviceId: device.deviceId}})+ // if (webSocket.readyState === WebSocket.OPEN) {+ // webSocket.send(frame)+ // }+}\ No newline at end of filediff --git a/freemocap-ui/src/services/supported_recorder.ts b/freemocap-ui/src/services/supported_recorder.tsnew file mode 100644index 00000000..6d704c32--- /dev/null+++ b/freemocap-ui/src/services/supported_recorder.ts@@ -0,0 +1,37 @@+export function supported() {+ console.log("Uncomment me to do useful stuff")+}+// function getSupportedMimeTypes(media, types, codecs) {+// const isSupported = MediaRecorder.isTypeSupported;+// const supported = [];+// types.forEach((type) => {+// const mimeType = `${media}/${type}`;+// codecs.forEach((codec) => [+// `${mimeType};codecs=${codec}`,+// `${mimeType};codecs=${codec.toUpperCase()}`,+// // /!\ false positive /!\+// // `${mimeType};codecs:${codec}`,+// // `${mimeType};codecs:${codec.toUpperCase()}`+// ].forEach(variation => {+// if(isSupported(variation))+// supported.push(variation);+// }));+// if (isSupported(mimeType))+// supported.push(mimeType);+// });+// return supported;+// };+//+// // Usage ------------------+//+// const videoTypes = ["webm", "ogg", "mp4", "x-matroska"];+// const audioTypes = ["webm", "ogg", "mp3", "x-matroska"];+// const codecs = ["should-not-be-supported","vp9", "vp9.0", "vp8", "vp8.0", "avc1", "av1", "h265", "h.265", "h264", "h.264", "opus", "pcm", "aac", "mpeg", "mp4a"];+//+// const supportedVideos = getSupportedMimeTypes("video", videoTypes, codecs);+// const supportedAudios = getSupportedMimeTypes("audio", audioTypes, codecs);+//+// console.log('-- Top supported Video : ', supportedVideos[0])+// console.log('-- Top supported Audio : ', supportedAudios[0])+// console.log('-- All supported Videos : ', supportedVideos)+// console.log('-- All supported Audios : ', supportedAudios)\ No newline at end of filediff --git a/requirements.txt b/requirements.txtindex d7ebd8e2..71a36ca4 100644--- a/requirements.txt+++ b/requirements.txt@@ -28,6 +28,8 @@ aiomultiprocess==0.9.0#viztracer==0.14.5pre-commit==2.17.0pandas==1.4.2+imageio+imageio-ffmpeg# Requirements for core-processorpyqtgraph==0.12.4pyqt5==5.15.6@@ -35,4 +37,5 @@ pyopengl==3.1.6mediapipe~=0.8.9.1setuptools~=58.0.4aniposelib~=0.4.3-rich~=12.3.0\ No newline at end of file+rich~=12.3.0+pyzmq\ No newline at end of filediff --git a/src/api/routes/camera/cam_data_ws.py b/src/api/routes/camera/cam_data_ws.pyindex 1db7f9c6..9be56b9c 100644--- a/src/api/routes/camera/cam_data_ws.py+++ b/src/api/routes/camera/cam_data_ws.py@@ -1,6 +1,8 @@import logging+from datetime import datetimeimport cv2+import numpy as npfrom fastapi import APIRouter, WebSocketfrom src.cameras.capture.dataclasses.frame_payload import FramePayload@@ -22,7 +24,11 @@ async def websocket_send(web_socket: WebSocket, input_payload: FramePayload):@cam_ws_router.websocket("/ws/hello_world")async def preview_webcam(web_socket: WebSocket):await web_socket.accept()- try:- await web_socket.send_json({"message": "Hi Nikki"})- except:- web_socket.close()+ while True:+ last_read = datetime.now()+ byte_data = await web_socket.receive_bytes()+ time_since = datetime.now()+ print("Per Frame Time: ", time_since - last_read)+ buffer = np.frombuffer(byte_data, dtype='uint8').reshape((600, 800, 4))+ buffer = cv2.cvtColor(buffer, cv2.COLOR_BGR2RGB)+diff --git a/src/api/routes/camera/camera_route.py b/src/api/routes/camera/camera_route.pyindex 81db2eb6..f239a52a 100644--- a/src/api/routes/camera/camera_route.py+++ b/src/api/routes/camera/camera_route.py@@ -1,8 +1,6 @@import logging-import tracebackimport cv2-import pyglview as pyglviewfrom fastapi import APIRouterfrom pydantic import BaseModel@@ -67,6 +65,7 @@ async def cv2_imshow_all_camera(camera_preview_model: CameraPreviewModel = Camershould_continue = Falsecv2.destroyWindow(cv2_window_name)+@camera_router.get("/camera/detect")async def get_cameras():return get_or_create_cams()diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.pyindex 90ed3ba7..539c9da4 100644--- a/src/cameras/capture/opencv_camera/opencv_camera.py+++ b/src/cameras/capture/opencv_camera/opencv_camera.py@@ -1,3 +1,4 @@+import asyncioimport loggingimport platformimport time@@ -7,6 +8,7 @@ import cv2from src.cameras.capture.dataclasses.frame_payload import FramePayloadfrom src.cameras.persistence.video_writer.video_recorder import VideoRecorder+from src.cameras.viewer.cv_cam_viewer import CvCamViewerfrom src.config.webcam_config import WebcamConfigfrom src.cameras.capture.opencv_camera.camera_stream_thread_handler import VideoCaptureThread@@ -187,6 +189,14 @@ class OpenCVCamera:def stop_frame_capture(self):self.close()+ async def show(self):+ viewer = CvCamViewer()+ viewer.begin_viewer(self.webcam_id_as_str)+ while True:+ if self.new_frame_ready:+ viewer.recv_img(self.latest_frame)+ await asyncio.sleep(0)+def close(self):try:self._running_thread.stop()diff --git a/src/cameras/utils/cap_connection_test.py b/src/cameras/utils/cap_connection_test.pynew file mode 100644index 00000000..e69de29bdiff --git a/src/cameras/viewer/cv_cam_viewer.py b/src/cameras/viewer/cv_cam_viewer.pynew file mode 100644index 00000000..de48b016--- /dev/null+++ b/src/cameras/viewer/cv_cam_viewer.py@@ -0,0 +1,33 @@+import multiprocessing+from multiprocessing import Process+from typing import Union++import cv2++from src.cameras.capture.dataclasses.frame_payload import FramePayload+++class CvCamViewer:+ def __init__(self):+ self._process: Process = None+ self._payload = None+ self._recv, self._send = multiprocessing.Pipe(duplex=False)++ @staticmethod+ def _begin(conn, webcam_id):+ while True:+ payload: FramePayload = conn.recv()+ if payload:+ cv2.imshow(str(webcam_id), payload.image)+ cv2.waitKey(1)++ def begin_viewer(self, webcam_id: Union[str, int]):+ self._process = Process(+ target=CvCamViewer._begin,+ args=(self._recv, webcam_id)+ )++ self._process.start()++ def recv_img(self, frame_payload):+ self._send.send(frame_payload)diff --git a/src/experiments/cv_cam_multi_process.py b/src/experiments/cv_cam_multi_process.pynew file mode 100644index 00000000..e7b1ce89--- /dev/null+++ b/src/experiments/cv_cam_multi_process.py@@ -0,0 +1,118 @@+import logging+import multiprocessing+import platform+import threading+import time+import traceback+from multiprocessing import Process, Queue+from typing import Union++import cv2++from src.cameras.capture.dataclasses.frame_payload import FramePayload+from src.cameras.capture.opencv_camera.opencv_camera import OpenCVCamera+from src.cameras.detection.cam_singleton import get_or_create_cams+from src.config.webcam_config import WebcamConfig++logger = logging.getLogger(__name__)+++class CvCamByProcess:++ def __init__(self, config: WebcamConfig = WebcamConfig()):+ self._process: Process = None+ self._recv, self._send = multiprocessing.Pipe(duplex=False)+ self._config = config+ self._is_capturing_frames = False+ self._new_frame_ready = False++ @property+ def webcam_id(self):+ return self._config.webcam_id++ @property+ def current_image(self) -> FramePayload:+ if self._is_capturing_frames:+ return self._recv.recv()+ return None++ def begin_frame_capture(self):+ if not self._is_capturing_frames:+ self._is_capturing_frames = True+ self._process = Process(+ target=self._start_frame_loop,+ args=(self._send,)+ )+ self._process.start()++ def end_frame_capture(self):+ self._is_capturing_frames = False+ self._process.kill()+ self._process = None++ def _create_obj(self, config: WebcamConfig):+ if platform.system() == "Windows":+ cap_backend = cv2.CAP_DSHOW+ else:+ cap_backend = cv2.CAP_ANY++ opencv_video_capture_object = cv2.VideoCapture(+ config.webcam_id, cap_backend+ )+ # self._apply_configuration()+ if not self._test_cap_connection(opencv_video_capture_object):+ raise Exception("Could not find camera")++ logger.debug(f"Camera found at port number {config.webcam_id}")+ fps_input_stream = int(opencv_video_capture_object.get(5))+ logger.debug(f"FPS of webcam hardware/input stream: {fps_input_stream}")+ return opencv_video_capture_object++ def _test_cap_connection(self, video_obj: cv2.VideoCapture):+ success, image = video_obj.read()++ if not success or image is None:+ logger.error(+ "Could not connect to a camera at port# {}".format(+ self._config.webcam_id+ )+ )+ return False++ return True++ def _start_frame_loop(self, conn):+ video_cap = self._create_obj(self._config)+ try:+ while self._is_capturing_frames:+ payload = self._get_next_frame(video_cap)+ conn.send(payload)+ except:+ logger.error("Frame loop thread exited due to error")+ traceback.print_exc()+ else:+ logger.info("Frame loop exited.")++ def _get_next_frame(self, video_cap: cv2.VideoCapture):+ # Why `grab()` not `read()`? see ->+ # https://stackoverflow.com/questions/57716962/difference-between-video-capture-read-and+ # -grab+ if not video_cap.grab():+ return FramePayload(False, None, None)++ timestamp_ns_pre = time.perf_counter_ns()+ success, image = video_cap.retrieve()+ timestamp_ns_post = time.perf_counter_ns()++ timestamp_ns = (timestamp_ns_pre + timestamp_ns_post) / 2++ self._new_frame_ready = success++ return FramePayload(success=success,+ image=image,+ timestamp=timestamp_ns)+++++diff --git a/src/experiments/viewers.py b/src/experiments/viewers.pynew file mode 100644index 00000000..6f11dfe9--- /dev/null+++ b/src/experiments/viewers.py@@ -0,0 +1,21 @@+import asyncio++from src.cameras.capture.opencv_camera.opencv_camera import OpenCVCamera+from src.cameras.detection.cam_singleton import get_or_create_cams+from src.config.webcam_config import WebcamConfig+++async def imshow_testing():+ cams = get_or_create_cams()+ cvcams = []+ for info in cams.cameras_found_list:+ c = OpenCVCamera(WebcamConfig(webcam_id=info.webcam_id))+ c.connect()+ c.start_frame_capture_thread()+ cvcams.append(c)++ await asyncio.gather(*[cam.show() for cam in cvcams])+++if __name__ == "__main__":+ asyncio.run(imshow_testing())