Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

OpenCVCamera comes with a viewer now with a "show" function Cv Cam has a multiprocess implementation in our experiments

Endurance Idehenca9167cfd8df
diff --git a/freemocap-ui/config-overrides.js b/freemocap-ui/config-overrides.js
new file mode 100644
index 00000000..00d9dc67
--- /dev/null
+++ b/freemocap-ui/config-overrides.js
@@ -0,0 +1,23 @@
+const webpack = require('webpack');
+module.exports = function override(config) {
+ const fallback = config.resolve.fallback || {};
+ Object.assign(fallback, {
+ "crypto": require.resolve("crypto-browserify"),
+ "stream": require.resolve("stream-browserify"),
+ "assert": require.resolve("assert"),
+ "http": require.resolve("stream-http"),
+ "https": require.resolve("https-browserify"),
+ "os": require.resolve("os-browserify"),
+ "url": require.resolve("url"),
+ fs: false,
+ path: false,
+ })
+ config.resolve.fallback = fallback;
+ config.plugins = (config.plugins || []).concat([
+ new webpack.ProvidePlugin({
+ process: 'process/browser',
+ Buffer: ['buffer', 'Buffer']
+ })
+ ])
+ return config;
+}
\ No newline at end of file
diff --git a/freemocap-ui/package.json b/freemocap-ui/package.json
index 1c1bf1ee..4e6c034e 100644
--- a/freemocap-ui/package.json
+++ b/freemocap-ui/package.json
@@ -14,7 +14,10 @@
"@mui/styled-engine": "^5.2.6",
"@mui/system": "^5.2.8",
"@reduxjs/toolkit": "^1.7.1",
+ "@shopify/react-web-worker": "^5.0.1",
+ "@techstark/opencv-js": "^4.5.5-release.2",
"@types/dom-mediacapture-record": "^1.0.11",
+ "@types/w3c-image-capture": "^1.0.6",
"@wmik/use-media-recorder": "^1.6.3-beta.0",
"axios": "^0.25.0",
"class-transformer": "0.5.1",
@@ -28,7 +31,7 @@
"react-scripts": "5.0.0",
"react-step-wizard": "^5.3.11",
"react-use": "^17.3.2",
- "react-use-websocket": "^2.9.1",
+ "react-use-websocket": "^3.0.0",
"react-use-wizard": "^2.2.0",
"react-webcam": "^6.0.0",
"reflect-metadata": "^0.1.13",
@@ -47,14 +50,24 @@
"@types/react": "^17.0.38",
"@types/react-dom": "^17.0.11",
"@types/react-redux": "^7.1.11",
- "@types/react-router-dom": "^5.1.6"
+ "@types/react-router-dom": "^5.1.6",
+ "assert": "^2.0.0",
+ "buffer": "^6.0.3",
+ "crypto-browserify": "^3.12.0",
+ "https-browserify": "^1.0.0",
+ "os-browserify": "^0.3.0",
+ "process": "^0.11.10",
+ "react-app-rewired": "^2.2.1",
+ "stream-browserify": "^3.0.0",
+ "stream-http": "^3.2.0",
+ "url": "^0.11.0"
},
"scripts": {
- "start": "react-scripts start",
"start:https": "HTTPS=true SSL_CRT_FILE=./localhost.pem SSL_KEY_FILE=./localhost-key.pem react-scripts start",
"certs": "./scripts/cert.sh",
- "build": "react-scripts build",
- "test": "react-scripts test",
+ "start": "react-app-rewired start",
+ "build": "react-app-rewired build",
+ "test": "react-app-rewired test",
"eject": "react-scripts eject"
},
"eslintConfig": {
diff --git a/freemocap-ui/public/index.html b/freemocap-ui/public/index.html
index 1dbdca65..b8c495c0 100644
--- a/freemocap-ui/public/index.html
+++ b/freemocap-ui/public/index.html
@@ -40,4 +40,7 @@
To create a production bundle, use `npm run build` or `yarn build`.
-->
</body>
+ <script type="text/javascript" src="https://unpkg.com/[email protected]/lib/opencv.js"></script>
+<!-- <script src="https://unpkg.com/opencv-javascript"></script>-->
+
</html>
diff --git a/freemocap-ui/src/components/webcam/LivePreview.tsx b/freemocap-ui/src/components/webcam/LivePreview.tsx
new file mode 100644
index 00000000..7e13d160
--- /dev/null
+++ b/freemocap-ui/src/components/webcam/LivePreview.tsx
@@ -0,0 +1,23 @@
+import React from "react";
+
+interface Props {
+ stream: MediaStream
+}
+
+export function LiveStreamPreview({ stream }) {
+ let videoPreviewRef = React.useRef<HTMLVideoElement>();
+
+ React.useEffect(() => {
+ if (videoPreviewRef.current && stream) {
+ // @ts-ignore
+ videoPreviewRef.current.srcObject = stream;
+ }
+ }, [stream]);
+
+ if (!stream) {
+ return null;
+ }
+
+ // @ts-ignore
+ return <video ref={videoPreviewRef} width={800} height={600} autoPlay />;
+}
diff --git a/freemocap-ui/src/components/webcam/webcam-capture.tsx b/freemocap-ui/src/components/webcam/webcam-capture.tsx
index 696e313f..95217baf 100644
--- a/freemocap-ui/src/components/webcam/webcam-capture.tsx
+++ b/freemocap-ui/src/components/webcam/webcam-capture.tsx
@@ -1,8 +1,7 @@
-import {Box} from "@mui/system";
-import React, {useState} from "react";
-import Webcam from "react-webcam";
-import useMediaRecorder from "@wmik/use-media-recorder";
-import {Chunk} from "../../services/Record";
+import React, {useEffect, useState} from "react";
+import {useDeviceStream} from "../../hooks/use-device-stream";
+import {Capture} from "../../services/Capture";
+import {Button} from "@mui/material";
interface Props {
device: MediaDeviceInfo
@@ -10,35 +9,30 @@ interface Props {
export const WebcamCapture = (props: Props) => {
const {device} = props
- const [dataAvailable, setDataAvailable] = useState<Chunk[]>([]);
- const {startRecording, status} = useMediaRecorder({
- onDataAvailable: (blob) => {
- const newArray = [...dataAvailable];
- newArray.push({frameData: blob, timestamp: Date.now()} as Chunk);
- setDataAvailable(newArray)
- console.log(newArray)
- },
- // blobOptions: {"type": "video\/mp4"},
- // onError: e => {
- // console.log(e)
- // },
- mediaStreamConstraints: {
- // video: true
- video: {
- deviceId: device.deviceId
- }
+ const videoRef = React.useRef<HTMLVideoElement>(null);
+ const stream = useDeviceStream(device)
+ // const { sendMessage, getWebSocket } = useWebSocket("ws://localhost:8080/ws/hello_world")
+ const [capture, setCapture] = useState<Capture>()
+ useEffect(() => {
+ const {current} = videoRef
+ if (current) {
+ // @ts-ignore
+ current.srcObject = stream
+ const capture = new Capture(videoRef)
+ setCapture(capture)
}
- });
+ }, [stream, videoRef])
- if (status === "idle") {
- console.log('start recording')
- startRecording(16.67)
+ if (!stream) {
+ return null
}
-
return (
- <Box>
- {device.label || `Device ${device.deviceId}`}
- <Webcam audio={true} videoConstraints={{deviceId: device.deviceId}} />
- </Box>
+ <>
+ <video ref={videoRef} height={600} width={800} autoPlay />
+ <canvas id={"canvasOutput"} />
+ <Button onClick={() => {
+ capture?.processVideo()
+ }}>Start Capture</Button>
+ </>
);
};
\ No newline at end of file
diff --git a/freemocap-ui/src/hooks/use-device-stream.ts b/freemocap-ui/src/hooks/use-device-stream.ts
new file mode 100644
index 00000000..05cd766b
--- /dev/null
+++ b/freemocap-ui/src/hooks/use-device-stream.ts
@@ -0,0 +1,12 @@
+import {useAsync} from "react-use";
+
+export const useDeviceStream = (device: MediaDeviceInfo) => {
+ const {value} = useAsync(async () => {
+ return await navigator.mediaDevices.getUserMedia({
+ video: {
+ deviceId: device.deviceId
+ },
+ })
+ }, [device])
+ return value
+}
\ No newline at end of file
diff --git a/freemocap-ui/src/services/Capture.tsx b/freemocap-ui/src/services/Capture.tsx
new file mode 100644
index 00000000..25332248
--- /dev/null
+++ b/freemocap-ui/src/services/Capture.tsx
@@ -0,0 +1,24 @@
+import cv from "@techstark/opencv-js";
+
+export class Capture {
+ private FPS = 60;
+ private _cap: cv.VideoCapture;
+ private _ws: WebSocket;
+
+ constructor(videoElement: any) {
+ const {height, width, current} = videoElement
+ console.log(height, width, current)
+ this._cap = new cv.VideoCapture(current);
+ this._ws = new WebSocket("ws://localhost:8080/ws/hello_world")
+ }
+
+ processVideo = () => {
+ setInterval(() => {
+ const image = new cv.Mat(600, 800, cv.CV_8UC4)
+ this._cap.read(image);
+ // cv.imshow('canvasOutput', image);
+ this._ws.send(image.data)
+ image.delete()
+ }, 10)
+ }
+}
\ No newline at end of file
diff --git a/freemocap-ui/src/services/image_send_worker.ts b/freemocap-ui/src/services/image_send_worker.ts
new file mode 100644
index 00000000..d70f20ae
--- /dev/null
+++ b/freemocap-ui/src/services/image_send_worker.ts
@@ -0,0 +1,16 @@
+const webSocket = new WebSocket("ws://localhost:8080/ws/hello_world")
+
+export const sendImageByRef = (camRef) => {
+ const imageSrc = camRef.current.getScreenshot({
+ height: 1080,
+ width: 1920,
+ });
+ webSocket.send(imageSrc)
+}
+
+export const sendImageByDevice = async (device: MediaDeviceInfo) => {
+ // const stream = await navigator.mediaDevices.getUserMedia({ video: { deviceId: device.deviceId}})
+ // if (webSocket.readyState === WebSocket.OPEN) {
+ // webSocket.send(frame)
+ // }
+}
\ No newline at end of file
diff --git a/freemocap-ui/src/services/supported_recorder.ts b/freemocap-ui/src/services/supported_recorder.ts
new file mode 100644
index 00000000..6d704c32
--- /dev/null
+++ b/freemocap-ui/src/services/supported_recorder.ts
@@ -0,0 +1,37 @@
+export function supported() {
+ console.log("Uncomment me to do useful stuff")
+}
+// function getSupportedMimeTypes(media, types, codecs) {
+// const isSupported = MediaRecorder.isTypeSupported;
+// const supported = [];
+// types.forEach((type) => {
+// const mimeType = `${media}/${type}`;
+// codecs.forEach((codec) => [
+// `${mimeType};codecs=${codec}`,
+// `${mimeType};codecs=${codec.toUpperCase()}`,
+// // /!\ false positive /!\
+// // `${mimeType};codecs:${codec}`,
+// // `${mimeType};codecs:${codec.toUpperCase()}`
+// ].forEach(variation => {
+// if(isSupported(variation))
+// supported.push(variation);
+// }));
+// if (isSupported(mimeType))
+// supported.push(mimeType);
+// });
+// return supported;
+// };
+//
+// // Usage ------------------
+//
+// const videoTypes = ["webm", "ogg", "mp4", "x-matroska"];
+// const audioTypes = ["webm", "ogg", "mp3", "x-matroska"];
+// const codecs = ["should-not-be-supported","vp9", "vp9.0", "vp8", "vp8.0", "avc1", "av1", "h265", "h.265", "h264", "h.264", "opus", "pcm", "aac", "mpeg", "mp4a"];
+//
+// const supportedVideos = getSupportedMimeTypes("video", videoTypes, codecs);
+// const supportedAudios = getSupportedMimeTypes("audio", audioTypes, codecs);
+//
+// console.log('-- Top supported Video : ', supportedVideos[0])
+// console.log('-- Top supported Audio : ', supportedAudios[0])
+// console.log('-- All supported Videos : ', supportedVideos)
+// console.log('-- All supported Audios : ', supportedAudios)
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index d7ebd8e2..71a36ca4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -28,6 +28,8 @@ aiomultiprocess==0.9.0
#viztracer==0.14.5
pre-commit==2.17.0
pandas==1.4.2
+imageio
+imageio-ffmpeg
# Requirements for core-processor
pyqtgraph==0.12.4
pyqt5==5.15.6
@@ -35,4 +37,5 @@ pyopengl==3.1.6
mediapipe~=0.8.9.1
setuptools~=58.0.4
aniposelib~=0.4.3
-rich~=12.3.0
\ No newline at end of file
+rich~=12.3.0
+pyzmq
\ No newline at end of file
diff --git a/src/api/routes/camera/cam_data_ws.py b/src/api/routes/camera/cam_data_ws.py
index 1db7f9c6..9be56b9c 100644
--- a/src/api/routes/camera/cam_data_ws.py
+++ b/src/api/routes/camera/cam_data_ws.py
@@ -1,6 +1,8 @@
import logging
+from datetime import datetime
import cv2
+import numpy as np
from fastapi import APIRouter, WebSocket
from src.cameras.capture.dataclasses.frame_payload import FramePayload
@@ -22,7 +24,11 @@ async def websocket_send(web_socket: WebSocket, input_payload: FramePayload):
@cam_ws_router.websocket("/ws/hello_world")
async def preview_webcam(web_socket: WebSocket):
await web_socket.accept()
- try:
- await web_socket.send_json({"message": "Hi Nikki"})
- except:
- web_socket.close()
+ while True:
+ last_read = datetime.now()
+ byte_data = await web_socket.receive_bytes()
+ time_since = datetime.now()
+ print("Per Frame Time: ", time_since - last_read)
+ buffer = np.frombuffer(byte_data, dtype='uint8').reshape((600, 800, 4))
+ buffer = cv2.cvtColor(buffer, cv2.COLOR_BGR2RGB)
+
diff --git a/src/api/routes/camera/camera_route.py b/src/api/routes/camera/camera_route.py
index 81db2eb6..f239a52a 100644
--- a/src/api/routes/camera/camera_route.py
+++ b/src/api/routes/camera/camera_route.py
@@ -1,8 +1,6 @@
import logging
-import traceback
import cv2
-import pyglview as pyglview
from fastapi import APIRouter
from pydantic import BaseModel
@@ -67,6 +65,7 @@ async def cv2_imshow_all_camera(camera_preview_model: CameraPreviewModel = Camer
should_continue = False
cv2.destroyWindow(cv2_window_name)
+
@camera_router.get("/camera/detect")
async def get_cameras():
return get_or_create_cams()
diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.py
index 90ed3ba7..539c9da4 100644
--- a/src/cameras/capture/opencv_camera/opencv_camera.py
+++ b/src/cameras/capture/opencv_camera/opencv_camera.py
@@ -1,3 +1,4 @@
+import asyncio
import logging
import platform
import time
@@ -7,6 +8,7 @@ import cv2
from src.cameras.capture.dataclasses.frame_payload import FramePayload
from src.cameras.persistence.video_writer.video_recorder import VideoRecorder
+from src.cameras.viewer.cv_cam_viewer import CvCamViewer
from src.config.webcam_config import WebcamConfig
from src.cameras.capture.opencv_camera.camera_stream_thread_handler import VideoCaptureThread
@@ -187,6 +189,14 @@ class OpenCVCamera:
def stop_frame_capture(self):
self.close()
+ async def show(self):
+ viewer = CvCamViewer()
+ viewer.begin_viewer(self.webcam_id_as_str)
+ while True:
+ if self.new_frame_ready:
+ viewer.recv_img(self.latest_frame)
+ await asyncio.sleep(0)
+
def close(self):
try:
self._running_thread.stop()
diff --git a/src/cameras/utils/cap_connection_test.py b/src/cameras/utils/cap_connection_test.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/cameras/viewer/cv_cam_viewer.py b/src/cameras/viewer/cv_cam_viewer.py
new file mode 100644
index 00000000..de48b016
--- /dev/null
+++ b/src/cameras/viewer/cv_cam_viewer.py
@@ -0,0 +1,33 @@
+import multiprocessing
+from multiprocessing import Process
+from typing import Union
+
+import cv2
+
+from src.cameras.capture.dataclasses.frame_payload import FramePayload
+
+
+class CvCamViewer:
+ def __init__(self):
+ self._process: Process = None
+ self._payload = None
+ self._recv, self._send = multiprocessing.Pipe(duplex=False)
+
+ @staticmethod
+ def _begin(conn, webcam_id):
+ while True:
+ payload: FramePayload = conn.recv()
+ if payload:
+ cv2.imshow(str(webcam_id), payload.image)
+ cv2.waitKey(1)
+
+ def begin_viewer(self, webcam_id: Union[str, int]):
+ self._process = Process(
+ target=CvCamViewer._begin,
+ args=(self._recv, webcam_id)
+ )
+
+ self._process.start()
+
+ def recv_img(self, frame_payload):
+ self._send.send(frame_payload)
diff --git a/src/experiments/cv_cam_multi_process.py b/src/experiments/cv_cam_multi_process.py
new file mode 100644
index 00000000..e7b1ce89
--- /dev/null
+++ b/src/experiments/cv_cam_multi_process.py
@@ -0,0 +1,118 @@
+import logging
+import multiprocessing
+import platform
+import threading
+import time
+import traceback
+from multiprocessing import Process, Queue
+from typing import Union
+
+import cv2
+
+from src.cameras.capture.dataclasses.frame_payload import FramePayload
+from src.cameras.capture.opencv_camera.opencv_camera import OpenCVCamera
+from src.cameras.detection.cam_singleton import get_or_create_cams
+from src.config.webcam_config import WebcamConfig
+
+logger = logging.getLogger(__name__)
+
+
+class CvCamByProcess:
+
+ def __init__(self, config: WebcamConfig = WebcamConfig()):
+ self._process: Process = None
+ self._recv, self._send = multiprocessing.Pipe(duplex=False)
+ self._config = config
+ self._is_capturing_frames = False
+ self._new_frame_ready = False
+
+ @property
+ def webcam_id(self):
+ return self._config.webcam_id
+
+ @property
+ def current_image(self) -> FramePayload:
+ if self._is_capturing_frames:
+ return self._recv.recv()
+ return None
+
+ def begin_frame_capture(self):
+ if not self._is_capturing_frames:
+ self._is_capturing_frames = True
+ self._process = Process(
+ target=self._start_frame_loop,
+ args=(self._send,)
+ )
+ self._process.start()
+
+ def end_frame_capture(self):
+ self._is_capturing_frames = False
+ self._process.kill()
+ self._process = None
+
+ def _create_obj(self, config: WebcamConfig):
+ if platform.system() == "Windows":
+ cap_backend = cv2.CAP_DSHOW
+ else:
+ cap_backend = cv2.CAP_ANY
+
+ opencv_video_capture_object = cv2.VideoCapture(
+ config.webcam_id, cap_backend
+ )
+ # self._apply_configuration()
+ if not self._test_cap_connection(opencv_video_capture_object):
+ raise Exception("Could not find camera")
+
+ logger.debug(f"Camera found at port number {config.webcam_id}")
+ fps_input_stream = int(opencv_video_capture_object.get(5))
+ logger.debug(f"FPS of webcam hardware/input stream: {fps_input_stream}")
+ return opencv_video_capture_object
+
+ def _test_cap_connection(self, video_obj: cv2.VideoCapture):
+ success, image = video_obj.read()
+
+ if not success or image is None:
+ logger.error(
+ "Could not connect to a camera at port# {}".format(
+ self._config.webcam_id
+ )
+ )
+ return False
+
+ return True
+
+ def _start_frame_loop(self, conn):
+ video_cap = self._create_obj(self._config)
+ try:
+ while self._is_capturing_frames:
+ payload = self._get_next_frame(video_cap)
+ conn.send(payload)
+ except:
+ logger.error("Frame loop thread exited due to error")
+ traceback.print_exc()
+ else:
+ logger.info("Frame loop exited.")
+
+ def _get_next_frame(self, video_cap: cv2.VideoCapture):
+ # Why `grab()` not `read()`? see ->
+ # https://stackoverflow.com/questions/57716962/difference-between-video-capture-read-and
+ # -grab
+ if not video_cap.grab():
+ return FramePayload(False, None, None)
+
+ timestamp_ns_pre = time.perf_counter_ns()
+ success, image = video_cap.retrieve()
+ timestamp_ns_post = time.perf_counter_ns()
+
+ timestamp_ns = (timestamp_ns_pre + timestamp_ns_post) / 2
+
+ self._new_frame_ready = success
+
+ return FramePayload(success=success,
+ image=image,
+ timestamp=timestamp_ns)
+
+
+
+
+
diff --git a/src/experiments/viewers.py b/src/experiments/viewers.py
new file mode 100644
index 00000000..6f11dfe9
--- /dev/null
+++ b/src/experiments/viewers.py
@@ -0,0 +1,21 @@
+import asyncio
+
+from src.cameras.capture.opencv_camera.opencv_camera import OpenCVCamera
+from src.cameras.detection.cam_singleton import get_or_create_cams
+from src.config.webcam_config import WebcamConfig
+
+
+async def imshow_testing():
+ cams = get_or_create_cams()
+ cvcams = []
+ for info in cams.cameras_found_list:
+ c = OpenCVCamera(WebcamConfig(webcam_id=info.webcam_id))
+ c.connect()
+ c.start_frame_capture_thread()
+ cvcams.append(c)
+
+ await asyncio.gather(*[cam.show() for cam in cvcams])
+
+
+if __name__ == "__main__":
+ asyncio.run(imshow_testing())