Skip to content

armgit / meet

Public Imported from GitHub
0

Open source video conferencing app powered by LiveKit. Built with Django and React.

✨(frontend) add mustache effects

Arnaud Robin1d5aebcfdc31
diff --git a/src/frontend/public/assets/mustache.png b/src/frontend/public/assets/mustache.png
new file mode 100644
index 00000000..09956bd4
Binary files /dev/null and b/src/frontend/public/assets/mustache.png differ
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts
index 6661c487..462ce624 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts
@@ -46,20 +46,25 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {
type: ProcessorType
- // Glasses image element
+ // Effect images
glassesImage?: HTMLImageElement
+ mustacheImage?: HTMLImageElement
constructor(opts: BackgroundOptions) {
this.name = 'face_landmarks'
this.options = opts
this.type = ProcessorType.FACE_LANDMARKS
- this._initGlassesImage()
+ this._initEffectImages()
}
- private _initGlassesImage() {
+ private _initEffectImages() {
this.glassesImage = new Image()
- this.glassesImage.src = '/assets/glasses.png' // You'll need to add this image to your public assets
+ this.glassesImage.src = '/assets/glasses.png'
this.glassesImage.crossOrigin = 'anonymous'
+
+ this.mustacheImage = new Image()
+ this.mustacheImage.src = '/assets/mustache.png'
+ this.mustacheImage.crossOrigin = 'anonymous'
}
static get isSupported() {
@@ -162,6 +167,53 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {
)
}
+ private drawEffect(
+ leftPoint: { x: number; y: number },
+ rightPoint: { x: number; y: number },
+ image: HTMLImageElement,
+ widthScale: number,
+ heightScale: number
+ ) {
+ // Calculate distance between points
+ const distance = Math.sqrt(
+ Math.pow(rightPoint.x - leftPoint.x, 2) +
+ Math.pow(rightPoint.y - leftPoint.y, 2)
+ )
+
+ // Scale image based on distance
+ const width = distance * PROCESSING_WIDTH * widthScale
+ const height = width * heightScale
+
+ // Calculate center position between points
+ const centerX = (leftPoint.x + rightPoint.x) / 2
+ const centerY = (leftPoint.y + rightPoint.y) / 2
+
+ // Draw image
+ this.outputCanvasCtx!.save()
+ this.outputCanvasCtx!.translate(
+ centerX * PROCESSING_WIDTH,
+ centerY * PROCESSING_HEIGHT
+ )
+
+ // Calculate rotation angle based on point positions
+ const angle = Math.atan2(
+ rightPoint.y - leftPoint.y,
+ rightPoint.x - leftPoint.x
+ )
+ this.outputCanvasCtx!.rotate(angle)
+
+ // Draw image centered at the midpoint between points
+ this.outputCanvasCtx!.drawImage(
+ image,
+ -width / 2,
+ -height / 2,
+ width,
+ height
+ )
+
+ this.outputCanvasCtx!.restore()
+ }
+
async drawFaceLandmarks() {
// Draw the original video frame at the canvas size
this.outputCanvasCtx!.drawImage(
@@ -185,49 +237,20 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {
this.outputCanvasCtx!.lineWidth = 2
for (const face of this.faceLandmarkerResult.faceLandmarks) {
- // Find eye landmarks (indices 33 and 263 are the left and right eye centers)
- const leftEye = face[33]
- const rightEye = face[263]
+ // Find eye landmarks (indices 33 and 263 are the left and right eye corners)
+ const leftEye = face[468]
+ const rightEye = face[473]
+
+ // Find mouth landmarks for mustache (indices 0 and 17 are the left and right corners of the mouth)
+ const leftMoustache = face[92]
+ const rightMoustache = face[322]
- if (leftEye && rightEye) {
- // Calculate glasses position and size
- const eyeDistance = Math.sqrt(
- Math.pow(rightEye.x - leftEye.x, 2) +
- Math.pow(rightEye.y - leftEye.y, 2)
- )
-
- // Scale glasses based on eye distance
- const glassesWidth = eyeDistance * PROCESSING_WIDTH * 2.5 // Adjust multiplier as needed
- const glassesHeight = glassesWidth * 0.7 // Adjust aspect ratio as needed
-
- // Calculate center position between eyes
- const centerX = (leftEye.x + rightEye.x) / 2
- const centerY = (leftEye.y + rightEye.y) / 2
-
- // Draw glasses
- this.outputCanvasCtx!.save()
- this.outputCanvasCtx!.translate(
- centerX * PROCESSING_WIDTH,
- centerY * PROCESSING_HEIGHT
- )
-
- // Calculate rotation angle based on eye positions
- const angle = Math.atan2(
- rightEye.y - leftEye.y,
- rightEye.x - leftEye.x
- )
- this.outputCanvasCtx!.rotate(angle)
-
- // Draw glasses centered at the midpoint between eyes
- this.outputCanvasCtx!.drawImage(
- this.glassesImage!,
- -glassesWidth / 2,
- -glassesHeight / 2,
- glassesWidth,
- glassesHeight
- )
-
- this.outputCanvasCtx!.restore()
+ if (leftEye && rightEye && this.options.showGlasses) {
+ this.drawEffect(leftEye, rightEye, this.glassesImage!, 2.5, 0.7)
+ }
+
+ if (leftMoustache && rightMoustache && this.options.showMustache) {
+ this.drawEffect(leftMoustache, rightMoustache, this.mustacheImage!, 1.5, 0.5)
}
}
}
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.ts
index 61220587..e7f27414 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts
@@ -8,7 +8,8 @@ import { FaceLandmarksProcessor } from './FaceLandmarksProcessor'
export type BackgroundOptions = {
blurRadius?: number
imagePath?: string
- showFaceLandmarks?: boolean
+ showGlasses?: boolean
+ showMustache?: boolean
}
export interface ProcessorSerialized {
diff --git a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx
index 306f16c0..10e19722 100644
--- a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx
+++ b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx
@@ -5,17 +5,17 @@ import {
BackgroundProcessorFactory,
BackgroundProcessorInterface,
ProcessorType,
+ BackgroundOptions
} from '../blur'
import { css } from '@/styled-system/css'
import { Text, P, ToggleButton, H } from '@/primitives'
import { styled } from '@/styled-system/jsx'
-import { BackgroundOptions } from '@livekit/track-processors'
import { BlurOn } from '@/components/icons/BlurOn'
import { BlurOnStrong } from '@/components/icons/BlurOnStrong'
import { useTrackToggle } from '@livekit/components-react'
import { Loader } from '@/primitives/Loader'
import { useSyncAfterDelay } from '@/hooks/useSyncAfterDelay'
-import { RiProhibited2Line, RiUserVoiceLine } from '@remixicon/react'
+import { RiProhibited2Line, RiGlassesLine, RiEmotionLine } from '@remixicon/react'
import { useHasFaceLandmarksAccess } from '../../hooks/useHasFaceLandmarksAccess'
enum BlurRadius {
@@ -141,9 +141,36 @@ export const EffectsConfiguration = ({
}
const tooltipLabel = (type: ProcessorType, options: BackgroundOptions) => {
+ if (type === ProcessorType.FACE_LANDMARKS) {
+ const effect = options.showGlasses ? 'glasses' : 'mustache'
+ return t(`faceLandmarks.${effect}.${isSelected(type, options) ? 'clear' : 'apply'}`)
+ }
return t(`${type}.${isSelected(type, options) ? 'clear' : 'apply'}`)
}
+ const getFaceLandmarksOptions = () => {
+ const processor = getProcessor()
+ if (processor?.serialize().type === ProcessorType.FACE_LANDMARKS) {
+ return processor.serialize().options as { showGlasses?: boolean; showMustache?: boolean }
+ }
+ return { showGlasses: false, showMustache: false }
+ }
+
+ const toggleFaceLandmarkEffect = async (effect: 'glasses' | 'mustache') => {
+ const currentOptions = getFaceLandmarksOptions()
+ const newOptions = {
+ ...currentOptions,
+ [effect === 'glasses' ? 'showGlasses' : 'showMustache']: !currentOptions[effect === 'glasses' ? 'showGlasses' : 'showMustache']
+ }
+
+ if (!newOptions.showGlasses && !newOptions.showMustache) {
+ // If both effects are off stop the processor
+ await clearEffect()
+ } else {
+ await toggleEffect(ProcessorType.FACE_LANDMARKS, newOptions)
+ }
+ }
+
return (
<div
className={css(
@@ -328,23 +355,36 @@ export const EffectsConfiguration = ({
<ToggleButton
variant="bigSquare"
aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
- blurRadius: 0,
+ showGlasses: true,
+ showMustache: false,
})}
tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
- blurRadius: 0,
+ showGlasses: true,
+ showMustache: false,
})}
isDisabled={processorPendingReveal}
- onChange={async () =>
- await toggleEffect(ProcessorType.FACE_LANDMARKS, {
- blurRadius: 0,
- })
- }
- isSelected={isSelected(ProcessorType.FACE_LANDMARKS, {
- blurRadius: 0,
+ onChange={async () => await toggleFaceLandmarkEffect('glasses')}
+ isSelected={getFaceLandmarksOptions().showGlasses}
+ data-attr="toggle-glasses"
+ >
+ <RiGlassesLine />
+ </ToggleButton>
+ <ToggleButton
+ variant="bigSquare"
+ aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
+ showGlasses: false,
+ showMustache: true,
+ })}
+ tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
+ showGlasses: false,
+ showMustache: true,
})}
- data-attr="toggle-face-landmarks"
+ isDisabled={processorPendingReveal}
+ onChange={async () => await toggleFaceLandmarkEffect('mustache')}
+ isSelected={getFaceLandmarksOptions().showMustache}
+ data-attr="toggle-mustache"
>
- <RiUserVoiceLine />
+ <RiEmotionLine />
</ToggleButton>
</div>
</div>
diff --git a/src/frontend/src/locales/de/rooms.json b/src/frontend/src/locales/de/rooms.json
index afd2198c..2a672fd7 100644
--- a/src/frontend/src/locales/de/rooms.json
+++ b/src/frontend/src/locales/de/rooms.json
@@ -151,9 +151,15 @@
"clear": "Virtuellen Hintergrund deaktivieren"
},
"faceLandmarks": {
- "title": "Gesichtsmerkmale",
- "apply": "Gesichtsmerkmale aktivieren",
- "clear": "Gesichtsmerkmale deaktivieren"
+ "title": "Visuelle Effekte",
+ "glasses": {
+ "apply": "Brille hinzufügen",
+ "clear": "Brille entfernen"
+ },
+ "mustache": {
+ "apply": "Schnurrbart hinzufügen",
+ "clear": "Schnurrbart entfernen"
+ }
},
"experimental": "Experimentelle Funktion. Eine v2 kommt für vollständige Browserunterstützung und verbesserte Qualität."
},
diff --git a/src/frontend/src/locales/en/rooms.json b/src/frontend/src/locales/en/rooms.json
index b94e9c70..23fc147d 100644
--- a/src/frontend/src/locales/en/rooms.json
+++ b/src/frontend/src/locales/en/rooms.json
@@ -150,9 +150,15 @@
"clear": "Disable virtual background"
},
"faceLandmarks": {
- "title": "Face landmarks",
- "apply": "Enable face landmarks",
- "clear": "Disable face landmarks"
+ "title": "Visual Effects",
+ "glasses": {
+ "apply": "Add Glasses",
+ "clear": "Remove Glasses"
+ },
+ "mustache": {
+ "apply": "Add Mustache",
+ "clear": "Remove Mustache"
+ }
},
"experimental": "Experimental feature. A v2 is coming for full browser support and improved quality."
},
diff --git a/src/frontend/src/locales/fr/rooms.json b/src/frontend/src/locales/fr/rooms.json
index 553de0c3..0508179c 100644
--- a/src/frontend/src/locales/fr/rooms.json
+++ b/src/frontend/src/locales/fr/rooms.json
@@ -150,9 +150,15 @@
"clear": "Désactiver l'arrière-plan virtuel"
},
"faceLandmarks": {
- "title": "Points du visage",
- "apply": "Activer les points du visage",
- "clear": "Désactiver les points du visage"
+ "title": "Effets visuels",
+ "glasses": {
+ "apply": "Ajouter des lunettes",
+ "clear": "Retirer les lunettes"
+ },
+ "mustache": {
+ "apply": "Ajouter une moustache",
+ "clear": "Retirer la moustache"
+ }
},
"experimental": "Fonctionnalité expérimentale. Une v2 arrive pour un support complet des navigateurs et une meilleure qualité."
},
diff --git a/src/frontend/src/locales/nl/rooms.json b/src/frontend/src/locales/nl/rooms.json
index 7494dde7..da56ae02 100644
--- a/src/frontend/src/locales/nl/rooms.json
+++ b/src/frontend/src/locales/nl/rooms.json
@@ -150,12 +150,14 @@
"clear": "Virtuele achtergrond uitschakelen"
},
"faceLandmarks": {
- "title": "Gezichtskenmerken",
- "apply": "Gezichtskenmerken inschakelen",
- "clear": "Gezichtskenmerken uitschakelen",
- "tooltip": {
- "apply": "Gezichtskenmerken inschakelen",
- "clear": "Gezichtskenmerken uitschakelen"
+ "title": "Visuele effecten",
+ "glasses": {
+ "apply": "Bril toevoegen",
+ "clear": "Bril verwijderen"
+ },
+ "mustache": {
+ "apply": "Snor toevoegen",
+ "clear": "Snor verwijderen"
}
},
"experimental": "Experimentele functie. Een v2 komt eraan voor volledige browserondersteuning en verbeterde kwaliteit."