Open source video conferencing app powered by LiveKit. Built with Django and React.
✨(frontend) add face tracking
Arnaud Robincd9b80b96617
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.tsnew file mode 100644index 00000000..da368f70--- /dev/null+++ b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts@@ -0,0 +1,251 @@+import { ProcessorOptions, Track } from 'livekit-client'+import posthog from 'posthog-js'+import {+ FilesetResolver,+ FaceLandmarker,+ FaceLandmarkerResult,+} from '@mediapipe/tasks-vision'+import {+ CLEAR_TIMEOUT,+ SET_TIMEOUT,+ TIMEOUT_TICK,+ timerWorkerScript,+} from './TimerWorker'+import {+ BackgroundProcessorInterface,+ BackgroundOptions,+ ProcessorType,+} from '.'++const PROCESSING_WIDTH = 256 * 3+const PROCESSING_HEIGHT = 144 * 3++const FACE_LANDMARKS_CANVAS_ID = 'face-landmarks-local'++export class FaceLandmarksProcessor implements BackgroundProcessorInterface {+ options: BackgroundOptions+ name: string+ processedTrack?: MediaStreamTrack | undefined++ source?: MediaStreamTrack+ sourceSettings?: MediaTrackSettings+ videoElement?: HTMLVideoElement+ videoElementLoaded?: boolean++ // Canvas containing the video processing result+ outputCanvas?: HTMLCanvasElement+ outputCanvasCtx?: CanvasRenderingContext2D++ faceLandmarker?: FaceLandmarker+ faceLandmarkerResult?: FaceLandmarkerResult++ // The resized image of the video source+ sourceImageData?: ImageData++ timerWorker?: Worker++ type: ProcessorType++ constructor(opts: BackgroundOptions) {+ this.name = 'face_landmarks'+ this.options = opts+ this.type = ProcessorType.FACE_LANDMARKS+ }++ static get isSupported() {+ return true // Face landmarks should work in all modern browsers+ }++ async init(opts: ProcessorOptions<Track.Kind>) {+ if (!opts.element) {+ throw new Error('Element is required for processing')+ }++ this.source = opts.track as MediaStreamTrack+ this.sourceSettings = this.source!.getSettings()+ this.videoElement = opts.element as HTMLVideoElement++ this._createMainCanvas()++ const stream = this.outputCanvas!.captureStream()+ const tracks = stream.getVideoTracks()+ if (tracks.length == 0) {+ throw new Error('No tracks found for processing')+ }+ this.processedTrack = tracks[0]++ await this.initFaceLandmarker()+ this._initWorker()++ posthog.capture('face-landmarks-init')+ }++ _initWorker() {+ this.timerWorker = new Worker(timerWorkerScript, {+ name: 'FaceLandmarks',+ })+ this.timerWorker.onmessage = (data) => this.onTimerMessage(data)+ if (this.videoElementLoaded) {+ this.timerWorker!.postMessage({+ id: SET_TIMEOUT,+ timeMs: 1000 / 30,+ })+ } else {+ this.videoElement!.onloadeddata = () => {+ this.videoElementLoaded = true+ this.timerWorker!.postMessage({+ id: SET_TIMEOUT,+ timeMs: 1000 / 30,+ })+ }+ }+ }++ onTimerMessage(response: { data: { id: number } }) {+ if (response.data.id === TIMEOUT_TICK) {+ this.process()+ }+ }++ async initFaceLandmarker() {+ const vision = await FilesetResolver.forVisionTasks(+ 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm'+ )+ this.faceLandmarker = await FaceLandmarker.createFromOptions(vision, {+ baseOptions: {+ modelAssetPath:+ 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/latest/face_landmarker.task',+ delegate: 'GPU',+ },+ runningMode: 'VIDEO',+ outputFaceBlendshapes: true,+ outputFacialTransformationMatrixes: true,+ })+ }++ async sizeSource() {+ this.outputCanvasCtx?.drawImage(+ this.videoElement!,+ 0,+ 0,+ this.videoElement!.videoWidth,+ this.videoElement!.videoHeight,+ 0,+ 0,+ PROCESSING_WIDTH,+ PROCESSING_HEIGHT+ )++ this.sourceImageData = this.outputCanvasCtx?.getImageData(+ 0,+ 0,+ PROCESSING_WIDTH,+ PROCESSING_HEIGHT+ )+ }++ async detectFaces() {+ const startTimeMs = performance.now()+ this.faceLandmarkerResult = this.faceLandmarker!.detectForVideo(+ this.sourceImageData!,+ startTimeMs+ )+ }++ async drawFaceLandmarks() {+ // Draw the original video frame at the canvas size+ this.outputCanvasCtx!.drawImage(+ this.videoElement!,+ 0,+ 0,+ this.videoElement!.videoWidth,+ this.videoElement!.videoHeight,+ 0,+ 0,+ PROCESSING_WIDTH,+ PROCESSING_HEIGHT+ )++ if (!this.faceLandmarkerResult?.faceLandmarks) {+ return+ }++ // Draw face landmarks+ this.outputCanvasCtx!.strokeStyle = '#00FF00'+ this.outputCanvasCtx!.lineWidth = 2++ for (const face of this.faceLandmarkerResult.faceLandmarks) {+ for (const landmark of face) {+ // Use the same dimensions as the canvas/video display size+ const x = landmark.x * PROCESSING_WIDTH+ const y = landmark.y * PROCESSING_HEIGHT++ this.outputCanvasCtx!.beginPath()+ this.outputCanvasCtx!.arc(x, y, 2, 0, 2 * Math.PI)+ this.outputCanvasCtx!.stroke()+ }+ }+ }++ async process() {+ await this.sizeSource()+ await this.detectFaces()+ await this.drawFaceLandmarks()++ this.timerWorker!.postMessage({+ id: SET_TIMEOUT,+ timeMs: 1000 / 30,+ })+ }++ _createMainCanvas() {+ this.outputCanvas = document.querySelector(+ `#${FACE_LANDMARKS_CANVAS_ID}`+ ) as HTMLCanvasElement+ if (!this.outputCanvas) {+ this.outputCanvas = this._createCanvas(+ FACE_LANDMARKS_CANVAS_ID,+ PROCESSING_WIDTH,+ PROCESSING_HEIGHT+ )+ }+ this.outputCanvasCtx = this.outputCanvas.getContext('2d')!+ }++ _createCanvas(id: string, width: number, height: number) {+ const element = document.createElement('canvas')+ element.setAttribute('id', id)+ element.setAttribute('width', '' + width)+ element.setAttribute('height', '' + height)+ return element+ }++ update(opts: BackgroundOptions): void {+ this.options = opts+ }++ async restart(opts: ProcessorOptions<Track.Kind>) {+ await this.destroy()+ return this.init(opts)+ }++ async destroy() {+ this.timerWorker?.postMessage({+ id: CLEAR_TIMEOUT,+ })++ this.timerWorker?.terminate()+ this.faceLandmarker?.close()+ }++ clone() {+ return new FaceLandmarksProcessor(this.options)+ }++ serialize() {+ return {+ type: this.type,+ options: this.options,+ }+ }+}\ No newline at end of filediff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.tsindex 893b9eb0..61220587 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts@@ -3,10 +3,12 @@ import { Track, TrackProcessor } from 'livekit-client'import { BackgroundBlurTrackProcessorJsWrapper } from './BackgroundBlurTrackProcessorJsWrapper'import { BackgroundCustomProcessor } from './BackgroundCustomProcessor'import { BackgroundVirtualTrackProcessorJsWrapper } from './BackgroundVirtualTrackProcessorJsWrapper'+import { FaceLandmarksProcessor } from './FaceLandmarksProcessor'export type BackgroundOptions = {blurRadius?: numberimagePath?: string+ showFaceLandmarks?: boolean}export interface ProcessorSerialized {@@ -25,11 +27,12 @@ export interface BackgroundProcessorInterfaceexport enum ProcessorType {BLUR = 'blur',VIRTUAL = 'virtual',+ FACE_LANDMARKS = 'faceLandmarks'}export class BackgroundProcessorFactory {static isSupported() {- return ProcessorWrapper.isSupported || BackgroundCustomProcessor.isSupported+ return ProcessorWrapper.isSupported || BackgroundCustomProcessor.isSupported || FaceLandmarksProcessor.isSupported}static getProcessor(@@ -50,6 +53,10 @@ export class BackgroundProcessorFactory {if (BackgroundCustomProcessor.isSupported) {return new BackgroundCustomProcessor(opts)}+ } else if (type === ProcessorType.FACE_LANDMARKS) {+ if (FaceLandmarksProcessor.isSupported) {+ return new FaceLandmarksProcessor(opts)+ }}return undefined}diff --git a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsxindex b5765ded..c0a59ef3 100644--- a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx+++ b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx@@ -15,7 +15,7 @@ import { BlurOnStrong } from '@/components/icons/BlurOnStrong'import { useTrackToggle } from '@livekit/components-react'import { Loader } from '@/primitives/Loader'import { useSyncAfterDelay } from '@/hooks/useSyncAfterDelay'-import { RiProhibited2Line } from '@remixicon/react'+import { RiProhibited2Line, RiUserVoiceLine } from '@remixicon/react'enum BlurRadius {NONE = 0,@@ -302,6 +302,49 @@ export const EffectsConfiguration = ({</ToggleButton></div></div>+ <div+ className={css({+ marginTop: '1.5rem',+ })}+ >+ <H+ lvl={3}+ style={{+ marginBottom: '1rem',+ }}+ variant="bodyXsBold"+ >+ {t('faceLandmarks.title')}+ </H>+ <div+ className={css({+ display: 'flex',+ gap: '1.25rem',+ })}+ >+ <ToggleButton+ variant="bigSquare"+ aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {+ blurRadius: 0,+ })}+ tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {+ blurRadius: 0,+ })}+ isDisabled={processorPendingReveal}+ onChange={async () =>+ await toggleEffect(ProcessorType.FACE_LANDMARKS, {+ blurRadius: 0,+ })+ }+ isSelected={isSelected(ProcessorType.FACE_LANDMARKS, {+ blurRadius: 0,+ })}+ data-attr="toggle-face-landmarks"+ >+ <RiUserVoiceLine />+ </ToggleButton>+ </div>+ </div><divclassName={css({marginTop: '1.5rem',diff --git a/src/frontend/src/locales/de/rooms.json b/src/frontend/src/locales/de/rooms.jsonindex b01114b6..afd2198c 100644--- a/src/frontend/src/locales/de/rooms.json+++ b/src/frontend/src/locales/de/rooms.json@@ -135,22 +135,27 @@}},"effects": {- "activateCamera": "",- "notAvailable": "",- "heading": "",+ "activateCamera": "Ihre Kamera ist deaktiviert. Wählen Sie eine Option, um sie zu aktivieren.",+ "notAvailable": "Videoeffekte werden in Kürze in Ihrem Browser verfügbar sein. Wir arbeiten daran! In der Zwischenzeit können Sie Google Chrome für beste Leistung oder Firefox verwenden :(",+ "heading": "Unschärfe","blur": {- "title": "",- "light": "",- "normal": "",- "apply": "",- "clear": ""+ "title": "Hintergrundunschärfe",+ "light": "Leichte Unschärfe",+ "normal": "Unschärfe",+ "apply": "Unschärfe aktivieren",+ "clear": "Unschärfe deaktivieren"},"virtual": {- "title": "",- "apply": "",- "clear": ""+ "title": "Virtueller Hintergrund",+ "apply": "Virtuellen Hintergrund aktivieren",+ "clear": "Virtuellen Hintergrund deaktivieren"+ },+ "faceLandmarks": {+ "title": "Gesichtsmerkmale",+ "apply": "Gesichtsmerkmale aktivieren",+ "clear": "Gesichtsmerkmale deaktivieren"},- "experimental": ""+ "experimental": "Experimentelle Funktion. Eine v2 kommt für vollständige Browserunterstützung und verbesserte Qualität."},"sidePanel": {"heading": {diff --git a/src/frontend/src/locales/en/rooms.json b/src/frontend/src/locales/en/rooms.jsonindex a0bdd2ae..b94e9c70 100644--- a/src/frontend/src/locales/en/rooms.json+++ b/src/frontend/src/locales/en/rooms.json@@ -149,6 +149,11 @@"apply": "Enable virtual background","clear": "Disable virtual background"},+ "faceLandmarks": {+ "title": "Face landmarks",+ "apply": "Enable face landmarks",+ "clear": "Disable face landmarks"+ },"experimental": "Experimental feature. A v2 is coming for full browser support and improved quality."},"sidePanel": {diff --git a/src/frontend/src/locales/fr/rooms.json b/src/frontend/src/locales/fr/rooms.jsonindex eda79c8c..553de0c3 100644--- a/src/frontend/src/locales/fr/rooms.json+++ b/src/frontend/src/locales/fr/rooms.json@@ -134,22 +134,27 @@}},"effects": {- "activateCamera": "Votre camera est désactivée. Choisissez une option pour l'activer.",- "notAvailable": "Les effets vidéo seront bientôt disponible sur votre navigateur. Nous y travaillons ! En attendant, vous pouvez utiliser Google Chrome pour une meilleure performance ou Firefox :(",+ "activateCamera": "Votre caméra est désactivée. Choisissez une option pour l'activer.",+ "notAvailable": "Les effets vidéo seront bientôt disponibles sur votre navigateur. Nous y travaillons ! En attendant, vous pouvez utiliser Google Chrome pour de meilleures performances ou Firefox :(","heading": "Flou","blur": {"title": "Flou d'arrière-plan",- "light": "Léger flou",+ "light": "Flou léger","normal": "Flou",- "apply": "Appliquer le flou",+ "apply": "Activer le flou","clear": "Désactiver le flou"},"virtual": {"title": "Arrière-plan virtuel",- "apply": "Appliquer l'arrière plan virtuel",- "clear": "Désactiver l'arrière plan virtuel"+ "apply": "Activer l'arrière-plan virtuel",+ "clear": "Désactiver l'arrière-plan virtuel"},- "experimental": "Fonctionnalité expérimentale. Une v2 arrive pour un support complet sur tous les navigateurs et une meilleur qualité."+ "faceLandmarks": {+ "title": "Points du visage",+ "apply": "Activer les points du visage",+ "clear": "Désactiver les points du visage"+ },+ "experimental": "Fonctionnalité expérimentale. Une v2 arrive pour un support complet des navigateurs et une meilleure qualité."},"sidePanel": {"heading": {diff --git a/src/frontend/src/locales/nl/rooms.json b/src/frontend/src/locales/nl/rooms.jsonindex d862bfdc..7494dde7 100644--- a/src/frontend/src/locales/nl/rooms.json+++ b/src/frontend/src/locales/nl/rooms.json@@ -134,22 +134,31 @@}},"effects": {- "activateCamera": "Uw camera is uitgeschakeld. Kies een optie om hem in te schakelen.",- "notAvailable": "Video-effecten zijn binnenkort beschikbaar in uw browser. We werken hier aan! Ondertussen kunt u Chrome gebruiken voor de beste prestaties, of Firefox :(",+ "activateCamera": "Uw camera is uitgeschakeld. Kies een optie om deze in te schakelen.",+ "notAvailable": "Video-effecten zijn binnenkort beschikbaar in uw browser. We werken eraan! In de tussentijd kunt u Google Chrome gebruiken voor de beste prestaties of Firefox :(","heading": "Vervaging","blur": {- "title": "Achtergrond vervagen",- "light": "Licht vervagen",- "normal": "Vervagen",- "apply": "Schakel vervaging in",- "clear": "Schakel vervaging uit"+ "title": "Achtergrondvervaging",+ "light": "Lichte vervaging",+ "normal": "Vervaging",+ "apply": "Vervaging inschakelen",+ "clear": "Vervaging uitschakelen"},"virtual": {"title": "Virtuele achtergrond",- "apply": "Schakel virtuele achtergrond in",- "clear": "Schakel virtuele achtergrond uit"+ "apply": "Virtuele achtergrond inschakelen",+ "clear": "Virtuele achtergrond uitschakelen"},- "experimental": "Experimentele functionaliteit. Een v2 komt er aan met volledige browser-ondersteuning en verbeterde kwaliteit."+ "faceLandmarks": {+ "title": "Gezichtskenmerken",+ "apply": "Gezichtskenmerken inschakelen",+ "clear": "Gezichtskenmerken uitschakelen",+ "tooltip": {+ "apply": "Gezichtskenmerken inschakelen",+ "clear": "Gezichtskenmerken uitschakelen"+ }+ },+ "experimental": "Experimentele functie. Een v2 komt eraan voor volledige browserondersteuning en verbeterde kwaliteit."},"sidePanel": {"heading": {