Skip to content

armgit / meet

Public Imported from GitHub
0

Open source video conferencing app powered by LiveKit. Built with Django and React.

♻️(frontend) inline model weights to avoid loading them from remote

lebaudantoineaabb7d629c5c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 16120c3e..51848936 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,6 +20,7 @@ and this project adheres to
- ⬆️(mail) update mjml to v5 and @html-to/text-cli
- 🚸(frontend) initialize the join input name with the persisted full name
- ♻️(frontend) refactor background processors to use the new API
+- ♻️(frontend) inline model weights to avoid loading them from remote
### Fixed
diff --git a/src/frontend/public/assets/mediapipe/models/face_landmarker.task b/src/frontend/public/assets/mediapipe/models/face_landmarker.task
new file mode 100644
index 00000000..c50c845d
Binary files /dev/null and b/src/frontend/public/assets/mediapipe/models/face_landmarker.task differ
diff --git a/src/frontend/public/assets/mediapipe/models/selfie_segmenter_landscape.tflite b/src/frontend/public/assets/mediapipe/models/selfie_segmenter_landscape.tflite
new file mode 100644
index 00000000..3a508f8d
Binary files /dev/null and b/src/frontend/public/assets/mediapipe/models/selfie_segmenter_landscape.tflite differ
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts
index d13b047b..95b5ed32 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts
@@ -13,6 +13,7 @@ import {
} from './TimerWorker'
import {
BackgroundProcessorInterface,
+ SELFIE_SEGMENTER_MODEL_PATH,
type ProcessorConfig,
type ProcessorType,
} from '.'
@@ -158,8 +159,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
)
this.imageSegmenter = await ImageSegmenter.createFromOptions(vision, {
baseOptions: {
- modelAssetPath:
- 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite',
+ modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,
delegate: 'CPU', // Use CPU for Firefox.
},
runningMode: 'VIDEO',
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts
index 7299ae44..b6f6e3f2 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts
@@ -11,7 +11,7 @@ import {
TIMEOUT_TICK,
timerWorkerScript,
} from './TimerWorker'
-import { ProcessorType } from '.'
+import { FACE_LANDMARKS_MODEL_PATH, ProcessorType } from '.'
const PROCESSING_WIDTH = 256 * 3
const PROCESSING_HEIGHT = 144 * 3
@@ -133,8 +133,7 @@ export class FaceLandmarksProcessor implements TrackProcessor<Track.Kind> {
)
this.faceLandmarker = await FaceLandmarker.createFromOptions(vision, {
baseOptions: {
- modelAssetPath:
- 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/latest/face_landmarker.task',
+ modelAssetPath: FACE_LANDMARKS_MODEL_PATH,
delegate: 'GPU',
},
runningMode: 'VIDEO',
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts
index 0bf9c4bb..237e790c 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts
@@ -7,6 +7,7 @@ import {
type ProcessorConfig,
BackgroundProcessorInterface,
ProcessorType,
+ SELFIE_SEGMENTER_MODEL_PATH,
} from '.'
export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInterface {
@@ -22,12 +23,18 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInter
this.processor = BackgroundProcessor({
mode: 'virtual-background',
imagePath: opts.imagePath,
+ assetPaths: {
+ modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,
+ },
})
} else if (opts.type === 'blur') {
this.processorType = ProcessorType.BLUR
this.processor = BackgroundProcessor({
mode: 'background-blur',
blurRadius: opts.blurRadius,
+ assetPaths: {
+ modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,
+ },
})
} else {
throw new Error(
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.ts
index f9e9c6d4..892af9bd 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts
@@ -4,6 +4,12 @@ import { BackgroundCustomProcessor } from './BackgroundCustomProcessor'
import { UnifiedBackgroundTrackProcessor } from './UnifiedBackgroundTrackProcessor'
import { FaceLandmarksOptions } from './FaceLandmarksProcessor'
+export const SELFIE_SEGMENTER_MODEL_PATH =
+ '/assets/mediapipe/models/selfie_segmenter_landscape.tflite'
+
+export const FACE_LANDMARKS_MODEL_PATH =
+ '/assets/mediapipe/models/face_landmarker.task'
+
export enum ProcessorType {
BLUR = 'blur',
VIRTUAL = 'virtual',