Open source video conferencing app powered by LiveKit. Built with Django and React.
✨(frontend) add mustache effects
Arnaud Robin1d5aebcfdc31
diff --git a/src/frontend/public/assets/mustache.png b/src/frontend/public/assets/mustache.pngnew file mode 100644index 00000000..09956bd4Binary files /dev/null and b/src/frontend/public/assets/mustache.png differdiff --git a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.tsindex 6661c487..462ce624 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts@@ -46,20 +46,25 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {type: ProcessorType- // Glasses image element+ // Effect imagesglassesImage?: HTMLImageElement+ mustacheImage?: HTMLImageElementconstructor(opts: BackgroundOptions) {this.name = 'face_landmarks'this.options = optsthis.type = ProcessorType.FACE_LANDMARKS- this._initGlassesImage()+ this._initEffectImages()}- private _initGlassesImage() {+ private _initEffectImages() {this.glassesImage = new Image()- this.glassesImage.src = '/assets/glasses.png' // You'll need to add this image to your public assets+ this.glassesImage.src = '/assets/glasses.png'this.glassesImage.crossOrigin = 'anonymous'++ this.mustacheImage = new Image()+ this.mustacheImage.src = '/assets/mustache.png'+ this.mustacheImage.crossOrigin = 'anonymous'}static get isSupported() {@@ -162,6 +167,53 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {)}+ private drawEffect(+ leftPoint: { x: number; y: number },+ rightPoint: { x: number; y: number },+ image: HTMLImageElement,+ widthScale: number,+ heightScale: number+ ) {+ // Calculate distance between points+ const distance = Math.sqrt(+ Math.pow(rightPoint.x - leftPoint.x, 2) ++ Math.pow(rightPoint.y - leftPoint.y, 2)+ )++ // Scale image based on distance+ const width = distance * PROCESSING_WIDTH * widthScale+ const height = width * heightScale++ // Calculate center position between points+ const centerX = (leftPoint.x + rightPoint.x) / 2+ const centerY = (leftPoint.y + rightPoint.y) / 2++ // Draw image+ this.outputCanvasCtx!.save()+ this.outputCanvasCtx!.translate(+ centerX * PROCESSING_WIDTH,+ centerY * PROCESSING_HEIGHT+ )++ // Calculate rotation angle based on point positions+ const angle = Math.atan2(+ rightPoint.y - leftPoint.y,+ rightPoint.x - leftPoint.x+ )+ this.outputCanvasCtx!.rotate(angle)++ // Draw image centered at the midpoint between points+ this.outputCanvasCtx!.drawImage(+ image,+ -width / 2,+ -height / 2,+ width,+ height+ )++ this.outputCanvasCtx!.restore()+ }+async drawFaceLandmarks() {// Draw the original video frame at the canvas sizethis.outputCanvasCtx!.drawImage(@@ -185,49 +237,20 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {this.outputCanvasCtx!.lineWidth = 2for (const face of this.faceLandmarkerResult.faceLandmarks) {- // Find eye landmarks (indices 33 and 263 are the left and right eye centers)- const leftEye = face[33]- const rightEye = face[263]+ // Find eye landmarks (indices 33 and 263 are the left and right eye corners)+ const leftEye = face[468]+ const rightEye = face[473]++ // Find mouth landmarks for mustache (indices 0 and 17 are the left and right corners of the mouth)+ const leftMoustache = face[92]+ const rightMoustache = face[322]- if (leftEye && rightEye) {- // Calculate glasses position and size- const eyeDistance = Math.sqrt(- Math.pow(rightEye.x - leftEye.x, 2) +- Math.pow(rightEye.y - leftEye.y, 2)- )-- // Scale glasses based on eye distance- const glassesWidth = eyeDistance * PROCESSING_WIDTH * 2.5 // Adjust multiplier as needed- const glassesHeight = glassesWidth * 0.7 // Adjust aspect ratio as needed-- // Calculate center position between eyes- const centerX = (leftEye.x + rightEye.x) / 2- const centerY = (leftEye.y + rightEye.y) / 2-- // Draw glasses- this.outputCanvasCtx!.save()- this.outputCanvasCtx!.translate(- centerX * PROCESSING_WIDTH,- centerY * PROCESSING_HEIGHT- )-- // Calculate rotation angle based on eye positions- const angle = Math.atan2(- rightEye.y - leftEye.y,- rightEye.x - leftEye.x- )- this.outputCanvasCtx!.rotate(angle)-- // Draw glasses centered at the midpoint between eyes- this.outputCanvasCtx!.drawImage(- this.glassesImage!,- -glassesWidth / 2,- -glassesHeight / 2,- glassesWidth,- glassesHeight- )-- this.outputCanvasCtx!.restore()+ if (leftEye && rightEye && this.options.showGlasses) {+ this.drawEffect(leftEye, rightEye, this.glassesImage!, 2.5, 0.7)+ }++ if (leftMoustache && rightMoustache && this.options.showMustache) {+ this.drawEffect(leftMoustache, rightMoustache, this.mustacheImage!, 1.5, 0.5)}}}diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.tsindex 61220587..e7f27414 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts@@ -8,7 +8,8 @@ import { FaceLandmarksProcessor } from './FaceLandmarksProcessor'export type BackgroundOptions = {blurRadius?: numberimagePath?: string- showFaceLandmarks?: boolean+ showGlasses?: boolean+ showMustache?: boolean}export interface ProcessorSerialized {diff --git a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsxindex 306f16c0..10e19722 100644--- a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx+++ b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx@@ -5,17 +5,17 @@ import {BackgroundProcessorFactory,BackgroundProcessorInterface,ProcessorType,+ BackgroundOptions} from '../blur'import { css } from '@/styled-system/css'import { Text, P, ToggleButton, H } from '@/primitives'import { styled } from '@/styled-system/jsx'-import { BackgroundOptions } from '@livekit/track-processors'import { BlurOn } from '@/components/icons/BlurOn'import { BlurOnStrong } from '@/components/icons/BlurOnStrong'import { useTrackToggle } from '@livekit/components-react'import { Loader } from '@/primitives/Loader'import { useSyncAfterDelay } from '@/hooks/useSyncAfterDelay'-import { RiProhibited2Line, RiUserVoiceLine } from '@remixicon/react'+import { RiProhibited2Line, RiGlassesLine, RiEmotionLine } from '@remixicon/react'import { useHasFaceLandmarksAccess } from '../../hooks/useHasFaceLandmarksAccess'enum BlurRadius {@@ -141,9 +141,36 @@ export const EffectsConfiguration = ({}const tooltipLabel = (type: ProcessorType, options: BackgroundOptions) => {+ if (type === ProcessorType.FACE_LANDMARKS) {+ const effect = options.showGlasses ? 'glasses' : 'mustache'+ return t(`faceLandmarks.${effect}.${isSelected(type, options) ? 'clear' : 'apply'}`)+ }return t(`${type}.${isSelected(type, options) ? 'clear' : 'apply'}`)}+ const getFaceLandmarksOptions = () => {+ const processor = getProcessor()+ if (processor?.serialize().type === ProcessorType.FACE_LANDMARKS) {+ return processor.serialize().options as { showGlasses?: boolean; showMustache?: boolean }+ }+ return { showGlasses: false, showMustache: false }+ }++ const toggleFaceLandmarkEffect = async (effect: 'glasses' | 'mustache') => {+ const currentOptions = getFaceLandmarksOptions()+ const newOptions = {+ ...currentOptions,+ [effect === 'glasses' ? 'showGlasses' : 'showMustache']: !currentOptions[effect === 'glasses' ? 'showGlasses' : 'showMustache']+ }++ if (!newOptions.showGlasses && !newOptions.showMustache) {+ // If both effects are off stop the processor+ await clearEffect()+ } else {+ await toggleEffect(ProcessorType.FACE_LANDMARKS, newOptions)+ }+ }+return (<divclassName={css(@@ -328,23 +355,36 @@ export const EffectsConfiguration = ({<ToggleButtonvariant="bigSquare"aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {- blurRadius: 0,+ showGlasses: true,+ showMustache: false,})}tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {- blurRadius: 0,+ showGlasses: true,+ showMustache: false,})}isDisabled={processorPendingReveal}- onChange={async () =>- await toggleEffect(ProcessorType.FACE_LANDMARKS, {- blurRadius: 0,- })- }- isSelected={isSelected(ProcessorType.FACE_LANDMARKS, {- blurRadius: 0,+ onChange={async () => await toggleFaceLandmarkEffect('glasses')}+ isSelected={getFaceLandmarksOptions().showGlasses}+ data-attr="toggle-glasses"+ >+ <RiGlassesLine />+ </ToggleButton>+ <ToggleButton+ variant="bigSquare"+ aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {+ showGlasses: false,+ showMustache: true,+ })}+ tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {+ showGlasses: false,+ showMustache: true,})}- data-attr="toggle-face-landmarks"+ isDisabled={processorPendingReveal}+ onChange={async () => await toggleFaceLandmarkEffect('mustache')}+ isSelected={getFaceLandmarksOptions().showMustache}+ data-attr="toggle-mustache">- <RiUserVoiceLine />+ <RiEmotionLine /></ToggleButton></div></div>diff --git a/src/frontend/src/locales/de/rooms.json b/src/frontend/src/locales/de/rooms.jsonindex afd2198c..2a672fd7 100644--- a/src/frontend/src/locales/de/rooms.json+++ b/src/frontend/src/locales/de/rooms.json@@ -151,9 +151,15 @@"clear": "Virtuellen Hintergrund deaktivieren"},"faceLandmarks": {- "title": "Gesichtsmerkmale",- "apply": "Gesichtsmerkmale aktivieren",- "clear": "Gesichtsmerkmale deaktivieren"+ "title": "Visuelle Effekte",+ "glasses": {+ "apply": "Brille hinzufügen",+ "clear": "Brille entfernen"+ },+ "mustache": {+ "apply": "Schnurrbart hinzufügen",+ "clear": "Schnurrbart entfernen"+ }},"experimental": "Experimentelle Funktion. Eine v2 kommt für vollständige Browserunterstützung und verbesserte Qualität."},diff --git a/src/frontend/src/locales/en/rooms.json b/src/frontend/src/locales/en/rooms.jsonindex b94e9c70..23fc147d 100644--- a/src/frontend/src/locales/en/rooms.json+++ b/src/frontend/src/locales/en/rooms.json@@ -150,9 +150,15 @@"clear": "Disable virtual background"},"faceLandmarks": {- "title": "Face landmarks",- "apply": "Enable face landmarks",- "clear": "Disable face landmarks"+ "title": "Visual Effects",+ "glasses": {+ "apply": "Add Glasses",+ "clear": "Remove Glasses"+ },+ "mustache": {+ "apply": "Add Mustache",+ "clear": "Remove Mustache"+ }},"experimental": "Experimental feature. A v2 is coming for full browser support and improved quality."},diff --git a/src/frontend/src/locales/fr/rooms.json b/src/frontend/src/locales/fr/rooms.jsonindex 553de0c3..0508179c 100644--- a/src/frontend/src/locales/fr/rooms.json+++ b/src/frontend/src/locales/fr/rooms.json@@ -150,9 +150,15 @@"clear": "Désactiver l'arrière-plan virtuel"},"faceLandmarks": {- "title": "Points du visage",- "apply": "Activer les points du visage",- "clear": "Désactiver les points du visage"+ "title": "Effets visuels",+ "glasses": {+ "apply": "Ajouter des lunettes",+ "clear": "Retirer les lunettes"+ },+ "mustache": {+ "apply": "Ajouter une moustache",+ "clear": "Retirer la moustache"+ }},"experimental": "Fonctionnalité expérimentale. Une v2 arrive pour un support complet des navigateurs et une meilleure qualité."},diff --git a/src/frontend/src/locales/nl/rooms.json b/src/frontend/src/locales/nl/rooms.jsonindex 7494dde7..da56ae02 100644--- a/src/frontend/src/locales/nl/rooms.json+++ b/src/frontend/src/locales/nl/rooms.json@@ -150,12 +150,14 @@"clear": "Virtuele achtergrond uitschakelen"},"faceLandmarks": {- "title": "Gezichtskenmerken",- "apply": "Gezichtskenmerken inschakelen",- "clear": "Gezichtskenmerken uitschakelen",- "tooltip": {- "apply": "Gezichtskenmerken inschakelen",- "clear": "Gezichtskenmerken uitschakelen"+ "title": "Visuele effecten",+ "glasses": {+ "apply": "Bril toevoegen",+ "clear": "Bril verwijderen"+ },+ "mustache": {+ "apply": "Snor toevoegen",+ "clear": "Snor verwijderen"}},"experimental": "Experimentele functie. Een v2 komt eraan voor volledige browserondersteuning en verbeterde kwaliteit."