Skip to content

armgit / meet

Public Imported from GitHub
0

Open source video conferencing app powered by LiveKit. Built with Django and React.

♻️(frontend) inline MediaPipe WASM modules to avoid loading from remote

lebaudantoine581e115c03e1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 68436d9b..d0793c03 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,7 @@ and this project adheres to
- 🚸(frontend) initialize the join input name with the persisted full name
- ♻️(frontend) refactor background processors to use the new API
- ♻️(frontend) inline model weights to avoid loading them from remote
+- ♻️(frontend) inline MediaPipe WASM modules to avoid loading from remote
### Fixed
diff --git a/src/frontend/package-lock.json b/src/frontend/package-lock.json
index f5cf0734..c25d89e1 100644
--- a/src/frontend/package-lock.json
+++ b/src/frontend/package-lock.json
@@ -15,6 +15,7 @@
"@livekit/components-react": "2.9.21",
"@livekit/components-styles": "1.2.0",
"@livekit/track-processors": "0.7.2",
+ "@mediapipe/tasks-vision": "0.10.14",
"@pandacss/preset-panda": "1.11.3",
"@react-types/overlays": "3.10.0",
"@remixicon/react": "4.9.0",
diff --git a/src/frontend/package.json b/src/frontend/package.json
index 93cafdf1..f81aa632 100644
--- a/src/frontend/package.json
+++ b/src/frontend/package.json
@@ -22,6 +22,7 @@
"@livekit/components-react": "2.9.21",
"@livekit/components-styles": "1.2.0",
"@livekit/track-processors": "0.7.2",
+ "@mediapipe/tasks-vision": "0.10.14",
"@pandacss/preset-panda": "1.11.3",
"@react-types/overlays": "3.10.0",
"@remixicon/react": "4.9.0",
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts
index 95b5ed32..d2ce7c28 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts
@@ -16,6 +16,7 @@ import {
SELFIE_SEGMENTER_MODEL_PATH,
type ProcessorConfig,
type ProcessorType,
+ MEDIAPIPE_PATH_WASM,
} from '.'
const PROCESSING_WIDTH = 256
@@ -154,9 +155,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
}
async initSegmenter() {
- const vision = await FilesetResolver.forVisionTasks(
- 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm'
- )
+ const vision = await FilesetResolver.forVisionTasks(MEDIAPIPE_PATH_WASM)
this.imageSegmenter = await ImageSegmenter.createFromOptions(vision, {
baseOptions: {
modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts
index b6f6e3f2..5b4873cf 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts
@@ -11,7 +11,11 @@ import {
TIMEOUT_TICK,
timerWorkerScript,
} from './TimerWorker'
-import { FACE_LANDMARKS_MODEL_PATH, ProcessorType } from '.'
+import {
+ FACE_LANDMARKS_MODEL_PATH,
+ ProcessorType,
+ MEDIAPIPE_PATH_WASM,
+} from '.'
const PROCESSING_WIDTH = 256 * 3
const PROCESSING_HEIGHT = 144 * 3
@@ -128,9 +132,7 @@ export class FaceLandmarksProcessor implements TrackProcessor<Track.Kind> {
}
async initFaceLandmarker() {
- const vision = await FilesetResolver.forVisionTasks(
- 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm'
- )
+ const vision = await FilesetResolver.forVisionTasks(MEDIAPIPE_PATH_WASM)
this.faceLandmarker = await FaceLandmarker.createFromOptions(vision, {
baseOptions: {
modelAssetPath: FACE_LANDMARKS_MODEL_PATH,
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts
index 237e790c..5087215f 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts
@@ -7,6 +7,7 @@ import {
type ProcessorConfig,
BackgroundProcessorInterface,
ProcessorType,
+ MEDIAPIPE_PATH_WASM,
SELFIE_SEGMENTER_MODEL_PATH,
} from '.'
@@ -24,6 +25,7 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInter
mode: 'virtual-background',
imagePath: opts.imagePath,
assetPaths: {
+ tasksVisionFileSet: MEDIAPIPE_PATH_WASM,
modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,
},
})
@@ -33,6 +35,7 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInter
mode: 'background-blur',
blurRadius: opts.blurRadius,
assetPaths: {
+ tasksVisionFileSet: MEDIAPIPE_PATH_WASM,
modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,
},
})
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.ts
index 892af9bd..e6399f83 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts
@@ -10,6 +10,8 @@ export const SELFIE_SEGMENTER_MODEL_PATH =
export const FACE_LANDMARKS_MODEL_PATH =
'/assets/mediapipe/models/face_landmarker.task'
+export const MEDIAPIPE_PATH_WASM = '/assets/mediapipe/wasm'
+
export enum ProcessorType {
BLUR = 'blur',
VIRTUAL = 'virtual',
diff --git a/src/frontend/vite.config.ts b/src/frontend/vite.config.ts
index 44fda9ea..fd12d6fb 100644
--- a/src/frontend/vite.config.ts
+++ b/src/frontend/vite.config.ts
@@ -2,6 +2,7 @@ import { defineConfig, loadEnv } from 'vite'
import react from '@vitejs/plugin-react'
import { visualizer } from 'rollup-plugin-visualizer'
import svgr from 'vite-plugin-svgr'
+import { viteStaticCopy } from 'vite-plugin-static-copy'
// https://vitejs.dev/config/
export default defineConfig(({ mode }) => {
@@ -18,6 +19,15 @@ export default defineConfig(({ mode }) => {
},
},
}),
+ viteStaticCopy({
+ targets: [
+ {
+ src: 'node_modules/@mediapipe/tasks-vision/wasm/*',
+ dest: 'assets/mediapipe/wasm',
+ rename: { stripBase: 4 },
+ },
+ ],
+ }),
env.VITE_ANALYZE === 'true' &&
visualizer({
open: true,