Open source video conferencing app powered by LiveKit. Built with Django and React.
♻️(frontend) inline model weights to avoid loading them from remote
lebaudantoineaabb7d629c5c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 16120c3e..51848936 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -20,6 +20,7 @@ and this project adheres to- ⬆️(mail) update mjml to v5 and @html-to/text-cli- 🚸(frontend) initialize the join input name with the persisted full name- ♻️(frontend) refactor background processors to use the new API+- ♻️(frontend) inline model weights to avoid loading them from remote### Fixeddiff --git a/src/frontend/public/assets/mediapipe/models/face_landmarker.task b/src/frontend/public/assets/mediapipe/models/face_landmarker.tasknew file mode 100644index 00000000..c50c845dBinary files /dev/null and b/src/frontend/public/assets/mediapipe/models/face_landmarker.task differdiff --git a/src/frontend/public/assets/mediapipe/models/selfie_segmenter_landscape.tflite b/src/frontend/public/assets/mediapipe/models/selfie_segmenter_landscape.tflitenew file mode 100644index 00000000..3a508f8dBinary files /dev/null and b/src/frontend/public/assets/mediapipe/models/selfie_segmenter_landscape.tflite differdiff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.tsindex d13b047b..95b5ed32 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts@@ -13,6 +13,7 @@ import {} from './TimerWorker'import {BackgroundProcessorInterface,+ SELFIE_SEGMENTER_MODEL_PATH,type ProcessorConfig,type ProcessorType,} from '.'@@ -158,8 +159,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {)this.imageSegmenter = await ImageSegmenter.createFromOptions(vision, {baseOptions: {- modelAssetPath:- 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite',+ modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,delegate: 'CPU', // Use CPU for Firefox.},runningMode: 'VIDEO',diff --git a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.tsindex 7299ae44..b6f6e3f2 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts@@ -11,7 +11,7 @@ import {TIMEOUT_TICK,timerWorkerScript,} from './TimerWorker'-import { ProcessorType } from '.'+import { FACE_LANDMARKS_MODEL_PATH, ProcessorType } from '.'const PROCESSING_WIDTH = 256 * 3const PROCESSING_HEIGHT = 144 * 3@@ -133,8 +133,7 @@ export class FaceLandmarksProcessor implements TrackProcessor<Track.Kind> {)this.faceLandmarker = await FaceLandmarker.createFromOptions(vision, {baseOptions: {- modelAssetPath:- 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/latest/face_landmarker.task',+ modelAssetPath: FACE_LANDMARKS_MODEL_PATH,delegate: 'GPU',},runningMode: 'VIDEO',diff --git a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.tsindex 0bf9c4bb..237e790c 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts@@ -7,6 +7,7 @@ import {type ProcessorConfig,BackgroundProcessorInterface,ProcessorType,+ SELFIE_SEGMENTER_MODEL_PATH,} from '.'export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInterface {@@ -22,12 +23,18 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInterthis.processor = BackgroundProcessor({mode: 'virtual-background',imagePath: opts.imagePath,+ assetPaths: {+ modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,+ },})} else if (opts.type === 'blur') {this.processorType = ProcessorType.BLURthis.processor = BackgroundProcessor({mode: 'background-blur',blurRadius: opts.blurRadius,+ assetPaths: {+ modelAssetPath: SELFIE_SEGMENTER_MODEL_PATH,+ },})} else {throw new Error(diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.tsindex f9e9c6d4..892af9bd 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts@@ -4,6 +4,12 @@ import { BackgroundCustomProcessor } from './BackgroundCustomProcessor'import { UnifiedBackgroundTrackProcessor } from './UnifiedBackgroundTrackProcessor'import { FaceLandmarksOptions } from './FaceLandmarksProcessor'+export const SELFIE_SEGMENTER_MODEL_PATH =+ '/assets/mediapipe/models/selfie_segmenter_landscape.tflite'++export const FACE_LANDMARKS_MODEL_PATH =+ '/assets/mediapipe/models/face_landmarker.task'+export enum ProcessorType {BLUR = 'blur',VIRTUAL = 'virtual',