Open source video conferencing app powered by LiveKit. Built with Django and React.
🏗️(frontend) decouple landmark processor from background processors
lebaudantoineb27c5e9b926b
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.tsindex 4084072c..7e59d334 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts@@ -1,4 +1,4 @@-import { ProcessorOptions, Track } from 'livekit-client'+import { ProcessorOptions, Track, TrackProcessor } from 'livekit-client'import posthog from 'posthog-js'import {FilesetResolver,@@ -11,19 +11,20 @@ import {TIMEOUT_TICK,timerWorkerScript,} from './TimerWorker'-import {- BackgroundProcessorInterface,- BackgroundOptions,- ProcessorType,-} from '.'+import { ProcessorType } from '.'const PROCESSING_WIDTH = 256 * 3const PROCESSING_HEIGHT = 144 * 3const FACE_LANDMARKS_CANVAS_ID = 'face-landmarks-local'-export class FaceLandmarksProcessor implements BackgroundProcessorInterface {- options: BackgroundOptions+export type FaceLandmarksOptions = {+ showGlasses: boolean+ showFrench: boolean+}++export class FaceLandmarksProcessor implements TrackProcessor<Track.Kind> {+ options: FaceLandmarksOptionsname: stringprocessedTrack?: MediaStreamTrack | undefined@@ -50,7 +51,7 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {glassesImage?: HTMLImageElementmustacheImage?: HTMLImageElementberetImage?: HTMLImageElement- constructor(opts: BackgroundOptions) {+ constructor(opts: FaceLandmarksOptions) {this.name = 'face_landmarks'this.options = optsthis.type = ProcessorType.FACE_LANDMARKS@@ -314,7 +315,7 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {return element}- update(opts: BackgroundOptions): void {+ update(opts: FaceLandmarksOptions): void {this.options = opts}diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.tsindex 4700c520..3d59ba84 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts@@ -3,13 +3,10 @@ import { Track, TrackProcessor } from 'livekit-client'import { BackgroundBlurTrackProcessorJsWrapper } from './BackgroundBlurTrackProcessorJsWrapper'import { BackgroundCustomProcessor } from './BackgroundCustomProcessor'import { BackgroundVirtualTrackProcessorJsWrapper } from './BackgroundVirtualTrackProcessorJsWrapper'-import { FaceLandmarksProcessor } from './FaceLandmarksProcessor'export type BackgroundOptions = {blurRadius?: numberimagePath?: string- showGlasses?: boolean- showFrench?: boolean}export interface ProcessorSerialized {@@ -37,11 +34,7 @@ export class BackgroundProcessorFactory {}static isSupported() {- return (- ProcessorWrapper.isSupported ||- BackgroundCustomProcessor.isSupported ||- FaceLandmarksProcessor.isSupported- )+ return ProcessorWrapper.isSupported || BackgroundCustomProcessor.isSupported}static getProcessor(@@ -62,10 +55,6 @@ export class BackgroundProcessorFactory {if (BackgroundCustomProcessor.isSupported) {return new BackgroundCustomProcessor(opts)}- } else if (type === ProcessorType.FACE_LANDMARKS) {- if (FaceLandmarksProcessor.isSupported) {- return new FaceLandmarksProcessor(opts)- }}return undefined}diff --git a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsxindex 5d7c8c0b..a2ef25c1 100644--- a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx+++ b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx@@ -15,11 +15,8 @@ import { BlurOnStrong } from '@/components/icons/BlurOnStrong'import { useTrackToggle } from '@livekit/components-react'import { Loader } from '@/primitives/Loader'import { useSyncAfterDelay } from '@/hooks/useSyncAfterDelay'-import {- RiProhibited2Line,- RiGlassesLine,- RiGoblet2Fill,-} from '@remixicon/react'+import { RiProhibited2Line } from '@remixicon/react'+import { FunnyEffects } from './FunnyEffects'import { useHasFaceLandmarksAccess } from '../../hooks/useHasFaceLandmarksAccess'enum BlurRadius {@@ -152,42 +149,9 @@ export const EffectsConfiguration = ({}const tooltipLabel = (type: ProcessorType, options: BackgroundOptions) => {- if (type === ProcessorType.FACE_LANDMARKS) {- const effect = options.showGlasses ? 'glasses' : 'french'- return t(- `faceLandmarks.${effect}.${isSelected(type, options) ? 'clear' : 'apply'}`- )- }return t(`${type}.${isSelected(type, options) ? 'clear' : 'apply'}`)}- const getFaceLandmarksOptions = () => {- const processor = getProcessor()- if (processor?.serialize().type === ProcessorType.FACE_LANDMARKS) {- return processor.serialize().options as {- showGlasses?: boolean- showFrench?: boolean- }- }- return { showGlasses: false, showFrench: false }- }-- const toggleFaceLandmarkEffect = async (effect: 'glasses' | 'french') => {- const currentOptions = getFaceLandmarksOptions()- const newOptions = {- ...currentOptions,- [effect === 'glasses' ? 'showGlasses' : 'showFrench']:- !currentOptions[effect === 'glasses' ? 'showGlasses' : 'showFrench'],- }-- if (!newOptions.showGlasses && !newOptions.showFrench) {- // If both effects are off stop the processor- await clearEffect()- } else {- await toggleEffect(ProcessorType.FACE_LANDMARKS, newOptions)- }- }-return (<divclassName={css(@@ -275,6 +239,13 @@ export const EffectsConfiguration = ({: {})}>+ {hasFaceLandmarksAccess && (+ <FunnyEffects+ videoTrack={videoTrack}+ isPending={processorPendingReveal}+ onPending={setProcessorPending}+ />+ )}{isSupported ? (<><div>@@ -347,8 +318,6 @@ export const EffectsConfiguration = ({<BlurOnStrong /></ToggleButton></div>- </div>- {hasFaceLandmarksAccess && (<divclassName={css({marginTop: '1.5rem',@@ -361,108 +330,48 @@ export const EffectsConfiguration = ({}}variant="bodyXsBold">- {t('faceLandmarks.title')}+ {t('virtual.title')}</H><divclassName={css({display: 'flex',gap: '1.25rem',+ flexWrap: 'wrap',})}>- <ToggleButton- variant="bigSquare"- aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {- showGlasses: true,- showFrench: false,- })}- tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {- showGlasses: true,- showFrench: false,- })}- isDisabled={processorPendingReveal}- onChange={async () =>- await toggleFaceLandmarkEffect('glasses')- }- isSelected={getFaceLandmarksOptions().showGlasses}- data-attr="toggle-glasses"- >- <RiGlassesLine />- </ToggleButton>- <ToggleButton- variant="bigSquare"- aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {- showGlasses: false,- showFrench: true,- })}- tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {- showGlasses: false,- showFrench: true,- })}- isDisabled={processorPendingReveal}- onChange={async () =>- await toggleFaceLandmarkEffect('french')- }- isSelected={getFaceLandmarksOptions().showFrench}- data-attr="toggle-french"- >- <RiGoblet2Fill />- </ToggleButton>- </div>- </div>- )}- <div- className={css({- marginTop: '1.5rem',- })}- >- <H- lvl={3}- style={{- marginBottom: '1rem',- }}- variant="bodyXsBold"- >- {t('virtual.title')}- </H>- <div- className={css({- display: 'flex',- gap: '1.25rem',- flexWrap: 'wrap',- })}- >- {[...Array(8).keys()].map((i) => {- const imagePath = `/assets/backgrounds/${i + 1}.jpg`- const thumbnailPath = `/assets/backgrounds/thumbnails/${i + 1}.jpg`- return (- <ToggleButton- key={i}- variant="bigSquare"- aria-label={tooltipLabel(ProcessorType.VIRTUAL, {- imagePath,- })}- tooltip={tooltipLabel(ProcessorType.VIRTUAL, {- imagePath,- })}- isDisabled={processorPendingReveal}- onChange={async () =>- await toggleEffect(ProcessorType.VIRTUAL, {+ {[...Array(8).keys()].map((i) => {+ const imagePath = `/assets/backgrounds/${i + 1}.jpg`+ const thumbnailPath = `/assets/backgrounds/thumbnails/${i + 1}.jpg`+ return (+ <ToggleButton+ key={i}+ variant="bigSquare"+ aria-label={tooltipLabel(ProcessorType.VIRTUAL, {imagePath,- })- }- isSelected={isSelected(ProcessorType.VIRTUAL, {- imagePath,- })}- className={css({- bgSize: 'cover',- })}- style={{- backgroundImage: `url(${thumbnailPath})`,- }}- data-attr={`toggle-virtual-${i}`}- />- )- })}+ })}+ tooltip={tooltipLabel(ProcessorType.VIRTUAL, {+ imagePath,+ })}+ isDisabled={processorPendingReveal}+ onChange={async () =>+ await toggleEffect(ProcessorType.VIRTUAL, {+ imagePath,+ })+ }+ isSelected={isSelected(ProcessorType.VIRTUAL, {+ imagePath,+ })}+ className={css({+ bgSize: 'cover',+ })}+ style={{+ backgroundImage: `url(${thumbnailPath})`,+ }}+ data-attr={`toggle-virtual-${i}`}+ />+ )+ })}+ </div></div></div></div>diff --git a/src/frontend/src/features/rooms/livekit/components/effects/FunnyEffects.tsx b/src/frontend/src/features/rooms/livekit/components/effects/FunnyEffects.tsxnew file mode 100644index 00000000..c19eb43a--- /dev/null+++ b/src/frontend/src/features/rooms/livekit/components/effects/FunnyEffects.tsx@@ -0,0 +1,120 @@+import { css } from '@/styled-system/css'+import { H, ToggleButton } from '@/primitives'+import { ProcessorType } from '../blur'+import { RiGlassesLine, RiGoblet2Fill } from '@remixicon/react'+import { useTranslation } from 'react-i18next'+import { FaceLandmarksProcessor } from '../blur/FaceLandmarksProcessor'+import { LocalVideoTrack } from 'livekit-client'++export type FunnyEffectsProps = {+ videoTrack: LocalVideoTrack+ isPending?: boolean+ onPending: (value: boolean) => void+}++export const FunnyEffects = ({+ videoTrack,+ isPending,+ onPending,+}: FunnyEffectsProps) => {+ const { t } = useTranslation('rooms', { keyPrefix: 'effects' })++ const getOptions = () => {+ const processor = videoTrack?.getProcessor() as FaceLandmarksProcessor+ if (!processor || processor.type != ProcessorType.FACE_LANDMARKS) {+ return {+ showGlasses: false,+ showFrench: false,+ }+ }+ return processor.serialize().options+ }++ const options = getOptions()++ const toggleFaceLandmarkEffect = async (+ showEffect: 'showGlasses' | 'showFrench'+ ) => {+ const options = getOptions()+ const processor = videoTrack?.getProcessor() as FaceLandmarksProcessor++ const newOptions = {+ ...options,+ [showEffect]: !options[showEffect],+ }++ onPending(true)++ try {+ if (!newOptions.showGlasses && !newOptions.showFrench) {+ await videoTrack.stopProcessor()+ } else if (options.showGlasses || options.showFrench) {+ await processor?.update(newOptions)+ } else {+ const newProcessor = new FaceLandmarksProcessor(newOptions)+ await videoTrack.setProcessor(newProcessor)+ }+ } catch (e) {+ console.error('could not update processor', e)+ } finally {+ onPending(false)+ }+ }++ const getLabelAction = (enabled: boolean) => (enabled ? 'clear' : 'apply')++ return (+ <div+ className={css({+ marginBottom: '1.5rem',+ })}+ >+ <H+ lvl={3}+ style={{+ marginBottom: '1rem',+ }}+ variant="bodyXsBold"+ >+ {t('faceLandmarks.title')}+ </H>+ <div+ className={css({+ display: 'flex',+ gap: '1.25rem',+ })}+ >+ <ToggleButton+ variant="bigSquare"+ aria-label={t(+ `faceLandmarks.glasses.${getLabelAction(options.showGlasses)}`+ )}+ tooltip={t(+ `faceLandmarks.glasses.${getLabelAction(options.showGlasses)}`+ )}+ isDisabled={isPending}+ onChange={async () => await toggleFaceLandmarkEffect('showGlasses')}+ isSelected={options.showGlasses}+ data-attr="toggle-glasses"+ >+ <RiGlassesLine />+ </ToggleButton>+ <ToggleButton+ variant="bigSquare"+ aria-label={t(+ `faceLandmarks.french.${getLabelAction(options.showFrench)}`+ )}+ tooltip={t(+ `faceLandmarks.french.${getLabelAction(options.showFrench)}`+ )}+ isDisabled={isPending}+ onChange={async () => await toggleFaceLandmarkEffect('showFrench')}+ isSelected={options.showFrench}+ data-attr="toggle-french"+ >+ <RiGoblet2Fill />+ </ToggleButton>+ </div>+ </div>+ )+}diff --git a/src/frontend/src/features/rooms/livekit/hooks/useKonami.ts b/src/frontend/src/features/rooms/livekit/hooks/useKonami.tsnew file mode 100644index 00000000..e69de29b