Open source video conferencing app powered by LiveKit. Built with Django and React.
♻️(frontend) inline MediaPipe WASM modules to avoid loading from remote
lebaudantoine581e115c03e1
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 68436d9b..d0793c03 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -22,6 +22,7 @@ and this project adheres to- 🚸(frontend) initialize the join input name with the persisted full name- ♻️(frontend) refactor background processors to use the new API- ♻️(frontend) inline model weights to avoid loading them from remote+- ♻️(frontend) inline MediaPipe WASM modules to avoid loading from remote### Fixeddiff --git a/src/frontend/package-lock.json b/src/frontend/package-lock.jsonindex f5cf0734..c25d89e1 100644--- a/src/frontend/package-lock.json+++ b/src/frontend/package-lock.json@@ -15,6 +15,7 @@"@livekit/components-react": "2.9.21","@livekit/components-styles": "1.2.0","@livekit/track-processors": "0.7.2",+ "@mediapipe/tasks-vision": "0.10.14","@pandacss/preset-panda": "1.11.3","@react-types/overlays": "3.10.0","@remixicon/react": "4.9.0",diff --git a/src/frontend/package.json b/src/frontend/package.jsonindex 93cafdf1..f81aa632 100644--- a/src/frontend/package.json+++ b/src/frontend/package.json@@ -22,6 +22,7 @@"@livekit/components-react": "2.9.21","@livekit/components-styles": "1.2.0","@livekit/track-processors": "0.7.2",+ "@mediapipe/tasks-vision": "0.10.14","@pandacss/preset-panda": "1.11.3","@react-types/overlays": "3.10.0","@remixicon/react": "4.9.0",diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.tsindex 95b5ed32..d2ce7c28 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts@@ -16,6 +16,7 @@ import {SELFIE_SEGMENTER_MODEL_PATH,type ProcessorConfig,type ProcessorType,+ MEDIAPIPE_PATH_WASM,} from '.'const PROCESSING_WIDTH = 256@@ -154,9 +155,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {}async initSegmenter() {- const vision = await FilesetResolver.forVisionTasks(- 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm'- )+ const vision = await FilesetResolver.forVisionTasks(MEDIAPIPE_PATH_WASM)this.imageSegmenter = await ImageSegmenter.createFromOptions(vision, {baseOptions: {modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,diff --git a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.tsindex b6f6e3f2..5b4873cf 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts@@ -11,7 +11,11 @@ import {TIMEOUT_TICK,timerWorkerScript,} from './TimerWorker'-import { FACE_LANDMARKS_MODEL_PATH, ProcessorType } from '.'+import {+ FACE_LANDMARKS_MODEL_PATH,+ ProcessorType,+ MEDIAPIPE_PATH_WASM,+} from '.'const PROCESSING_WIDTH = 256 * 3const PROCESSING_HEIGHT = 144 * 3@@ -128,9 +132,7 @@ export class FaceLandmarksProcessor implements TrackProcessor<Track.Kind> {}async initFaceLandmarker() {- const vision = await FilesetResolver.forVisionTasks(- 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm'- )+ const vision = await FilesetResolver.forVisionTasks(MEDIAPIPE_PATH_WASM)this.faceLandmarker = await FaceLandmarker.createFromOptions(vision, {baseOptions: {modelAssetPath: FACE_LANDMARKS_MODEL_PATH,diff --git a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.tsindex 237e790c..5087215f 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts@@ -7,6 +7,7 @@ import {type ProcessorConfig,BackgroundProcessorInterface,ProcessorType,+ MEDIAPIPE_PATH_WASM,SELFIE_SEGMENTER_MODEL_PATH,} from '.'@@ -24,6 +25,7 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorIntermode: 'virtual-background',imagePath: opts.imagePath,assetPaths: {+ tasksVisionFileSet: MEDIAPIPE_PATH_WASM,modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,},})@@ -33,6 +35,7 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorIntermode: 'background-blur',blurRadius: opts.blurRadius,assetPaths: {+ tasksVisionFileSet: MEDIAPIPE_PATH_WASM,modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,},})diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.tsindex 892af9bd..e6399f83 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts@@ -10,6 +10,8 @@ export const SELFIE_SEGMENTER_MODEL_PATH =export const FACE_LANDMARKS_MODEL_PATH ='/assets/mediapipe/models/face_landmarker.task'+export const MEDIAPIPE_PATH_WASM = '/assets/mediapipe/wasm'+export enum ProcessorType {BLUR = 'blur',VIRTUAL = 'virtual',diff --git a/src/frontend/vite.config.ts b/src/frontend/vite.config.tsindex 44fda9ea..fd12d6fb 100644--- a/src/frontend/vite.config.ts+++ b/src/frontend/vite.config.ts@@ -2,6 +2,7 @@ import { defineConfig, loadEnv } from 'vite'import react from '@vitejs/plugin-react'import { visualizer } from 'rollup-plugin-visualizer'import svgr from 'vite-plugin-svgr'+import { viteStaticCopy } from 'vite-plugin-static-copy'// https://vitejs.dev/config/export default defineConfig(({ mode }) => {@@ -18,6 +19,15 @@ export default defineConfig(({ mode }) => {},},}),+ viteStaticCopy({+ targets: [+ {+ src: 'node_modules/@mediapipe/tasks-vision/wasm/*',+ dest: 'assets/mediapipe/wasm',+ rename: { stripBase: 4 },+ },+ ],+ }),env.VITE_ANALYZE === 'true' &&visualizer({open: true,