Skip to content

armgit / meet

Public Imported from GitHub
0

Open source video conferencing app powered by LiveKit. Built with Django and React.

🏗️(frontend) decouple landmark processor from background processors

lebaudantoineb27c5e9b926b
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts
index 4084072c..7e59d334 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/FaceLandmarksProcessor.ts
@@ -1,4 +1,4 @@
-import { ProcessorOptions, Track } from 'livekit-client'
+import { ProcessorOptions, Track, TrackProcessor } from 'livekit-client'
import posthog from 'posthog-js'
import {
FilesetResolver,
@@ -11,19 +11,20 @@ import {
TIMEOUT_TICK,
timerWorkerScript,
} from './TimerWorker'
-import {
- BackgroundProcessorInterface,
- BackgroundOptions,
- ProcessorType,
-} from '.'
+import { ProcessorType } from '.'
const PROCESSING_WIDTH = 256 * 3
const PROCESSING_HEIGHT = 144 * 3
const FACE_LANDMARKS_CANVAS_ID = 'face-landmarks-local'
-export class FaceLandmarksProcessor implements BackgroundProcessorInterface {
- options: BackgroundOptions
+export type FaceLandmarksOptions = {
+ showGlasses: boolean
+ showFrench: boolean
+}
+
+export class FaceLandmarksProcessor implements TrackProcessor<Track.Kind> {
+ options: FaceLandmarksOptions
name: string
processedTrack?: MediaStreamTrack | undefined
@@ -50,7 +51,7 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {
glassesImage?: HTMLImageElement
mustacheImage?: HTMLImageElement
beretImage?: HTMLImageElement
- constructor(opts: BackgroundOptions) {
+ constructor(opts: FaceLandmarksOptions) {
this.name = 'face_landmarks'
this.options = opts
this.type = ProcessorType.FACE_LANDMARKS
@@ -314,7 +315,7 @@ export class FaceLandmarksProcessor implements BackgroundProcessorInterface {
return element
}
- update(opts: BackgroundOptions): void {
+ update(opts: FaceLandmarksOptions): void {
this.options = opts
}
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.ts
index 4700c520..3d59ba84 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts
@@ -3,13 +3,10 @@ import { Track, TrackProcessor } from 'livekit-client'
import { BackgroundBlurTrackProcessorJsWrapper } from './BackgroundBlurTrackProcessorJsWrapper'
import { BackgroundCustomProcessor } from './BackgroundCustomProcessor'
import { BackgroundVirtualTrackProcessorJsWrapper } from './BackgroundVirtualTrackProcessorJsWrapper'
-import { FaceLandmarksProcessor } from './FaceLandmarksProcessor'
export type BackgroundOptions = {
blurRadius?: number
imagePath?: string
- showGlasses?: boolean
- showFrench?: boolean
}
export interface ProcessorSerialized {
@@ -37,11 +34,7 @@ export class BackgroundProcessorFactory {
}
static isSupported() {
- return (
- ProcessorWrapper.isSupported ||
- BackgroundCustomProcessor.isSupported ||
- FaceLandmarksProcessor.isSupported
- )
+ return ProcessorWrapper.isSupported || BackgroundCustomProcessor.isSupported
}
static getProcessor(
@@ -62,10 +55,6 @@ export class BackgroundProcessorFactory {
if (BackgroundCustomProcessor.isSupported) {
return new BackgroundCustomProcessor(opts)
}
- } else if (type === ProcessorType.FACE_LANDMARKS) {
- if (FaceLandmarksProcessor.isSupported) {
- return new FaceLandmarksProcessor(opts)
- }
}
return undefined
}
diff --git a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx
index 5d7c8c0b..a2ef25c1 100644
--- a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx
+++ b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx
@@ -15,11 +15,8 @@ import { BlurOnStrong } from '@/components/icons/BlurOnStrong'
import { useTrackToggle } from '@livekit/components-react'
import { Loader } from '@/primitives/Loader'
import { useSyncAfterDelay } from '@/hooks/useSyncAfterDelay'
-import {
- RiProhibited2Line,
- RiGlassesLine,
- RiGoblet2Fill,
-} from '@remixicon/react'
+import { RiProhibited2Line } from '@remixicon/react'
+import { FunnyEffects } from './FunnyEffects'
import { useHasFaceLandmarksAccess } from '../../hooks/useHasFaceLandmarksAccess'
enum BlurRadius {
@@ -152,42 +149,9 @@ export const EffectsConfiguration = ({
}
const tooltipLabel = (type: ProcessorType, options: BackgroundOptions) => {
- if (type === ProcessorType.FACE_LANDMARKS) {
- const effect = options.showGlasses ? 'glasses' : 'french'
- return t(
- `faceLandmarks.${effect}.${isSelected(type, options) ? 'clear' : 'apply'}`
- )
- }
return t(`${type}.${isSelected(type, options) ? 'clear' : 'apply'}`)
}
- const getFaceLandmarksOptions = () => {
- const processor = getProcessor()
- if (processor?.serialize().type === ProcessorType.FACE_LANDMARKS) {
- return processor.serialize().options as {
- showGlasses?: boolean
- showFrench?: boolean
- }
- }
- return { showGlasses: false, showFrench: false }
- }
-
- const toggleFaceLandmarkEffect = async (effect: 'glasses' | 'french') => {
- const currentOptions = getFaceLandmarksOptions()
- const newOptions = {
- ...currentOptions,
- [effect === 'glasses' ? 'showGlasses' : 'showFrench']:
- !currentOptions[effect === 'glasses' ? 'showGlasses' : 'showFrench'],
- }
-
- if (!newOptions.showGlasses && !newOptions.showFrench) {
- // If both effects are off stop the processor
- await clearEffect()
- } else {
- await toggleEffect(ProcessorType.FACE_LANDMARKS, newOptions)
- }
- }
-
return (
<div
className={css(
@@ -275,6 +239,13 @@ export const EffectsConfiguration = ({
: {}
)}
>
+ {hasFaceLandmarksAccess && (
+ <FunnyEffects
+ videoTrack={videoTrack}
+ isPending={processorPendingReveal}
+ onPending={setProcessorPending}
+ />
+ )}
{isSupported ? (
<>
<div>
@@ -347,8 +318,6 @@ export const EffectsConfiguration = ({
<BlurOnStrong />
</ToggleButton>
</div>
- </div>
- {hasFaceLandmarksAccess && (
<div
className={css({
marginTop: '1.5rem',
@@ -361,108 +330,48 @@ export const EffectsConfiguration = ({
}}
variant="bodyXsBold"
>
- {t('faceLandmarks.title')}
+ {t('virtual.title')}
</H>
<div
className={css({
display: 'flex',
gap: '1.25rem',
+ flexWrap: 'wrap',
})}
>
- <ToggleButton
- variant="bigSquare"
- aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
- showGlasses: true,
- showFrench: false,
- })}
- tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
- showGlasses: true,
- showFrench: false,
- })}
- isDisabled={processorPendingReveal}
- onChange={async () =>
- await toggleFaceLandmarkEffect('glasses')
- }
- isSelected={getFaceLandmarksOptions().showGlasses}
- data-attr="toggle-glasses"
- >
- <RiGlassesLine />
- </ToggleButton>
- <ToggleButton
- variant="bigSquare"
- aria-label={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
- showGlasses: false,
- showFrench: true,
- })}
- tooltip={tooltipLabel(ProcessorType.FACE_LANDMARKS, {
- showGlasses: false,
- showFrench: true,
- })}
- isDisabled={processorPendingReveal}
- onChange={async () =>
- await toggleFaceLandmarkEffect('french')
- }
- isSelected={getFaceLandmarksOptions().showFrench}
- data-attr="toggle-french"
- >
- <RiGoblet2Fill />
- </ToggleButton>
- </div>
- </div>
- )}
- <div
- className={css({
- marginTop: '1.5rem',
- })}
- >
- <H
- lvl={3}
- style={{
- marginBottom: '1rem',
- }}
- variant="bodyXsBold"
- >
- {t('virtual.title')}
- </H>
- <div
- className={css({
- display: 'flex',
- gap: '1.25rem',
- flexWrap: 'wrap',
- })}
- >
- {[...Array(8).keys()].map((i) => {
- const imagePath = `/assets/backgrounds/${i + 1}.jpg`
- const thumbnailPath = `/assets/backgrounds/thumbnails/${i + 1}.jpg`
- return (
- <ToggleButton
- key={i}
- variant="bigSquare"
- aria-label={tooltipLabel(ProcessorType.VIRTUAL, {
- imagePath,
- })}
- tooltip={tooltipLabel(ProcessorType.VIRTUAL, {
- imagePath,
- })}
- isDisabled={processorPendingReveal}
- onChange={async () =>
- await toggleEffect(ProcessorType.VIRTUAL, {
+ {[...Array(8).keys()].map((i) => {
+ const imagePath = `/assets/backgrounds/${i + 1}.jpg`
+ const thumbnailPath = `/assets/backgrounds/thumbnails/${i + 1}.jpg`
+ return (
+ <ToggleButton
+ key={i}
+ variant="bigSquare"
+ aria-label={tooltipLabel(ProcessorType.VIRTUAL, {
imagePath,
- })
- }
- isSelected={isSelected(ProcessorType.VIRTUAL, {
- imagePath,
- })}
- className={css({
- bgSize: 'cover',
- })}
- style={{
- backgroundImage: `url(${thumbnailPath})`,
- }}
- data-attr={`toggle-virtual-${i}`}
- />
- )
- })}
+ })}
+ tooltip={tooltipLabel(ProcessorType.VIRTUAL, {
+ imagePath,
+ })}
+ isDisabled={processorPendingReveal}
+ onChange={async () =>
+ await toggleEffect(ProcessorType.VIRTUAL, {
+ imagePath,
+ })
+ }
+ isSelected={isSelected(ProcessorType.VIRTUAL, {
+ imagePath,
+ })}
+ className={css({
+ bgSize: 'cover',
+ })}
+ style={{
+ backgroundImage: `url(${thumbnailPath})`,
+ }}
+ data-attr={`toggle-virtual-${i}`}
+ />
+ )
+ })}
+ </div>
</div>
</div>
</div>
diff --git a/src/frontend/src/features/rooms/livekit/components/effects/FunnyEffects.tsx b/src/frontend/src/features/rooms/livekit/components/effects/FunnyEffects.tsx
new file mode 100644
index 00000000..c19eb43a
--- /dev/null
+++ b/src/frontend/src/features/rooms/livekit/components/effects/FunnyEffects.tsx
@@ -0,0 +1,120 @@
+import { css } from '@/styled-system/css'
+import { H, ToggleButton } from '@/primitives'
+import { ProcessorType } from '../blur'
+import { RiGlassesLine, RiGoblet2Fill } from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
+import { FaceLandmarksProcessor } from '../blur/FaceLandmarksProcessor'
+import { LocalVideoTrack } from 'livekit-client'
+
+export type FunnyEffectsProps = {
+ videoTrack: LocalVideoTrack
+ isPending?: boolean
+ onPending: (value: boolean) => void
+}
+
+export const FunnyEffects = ({
+ videoTrack,
+ isPending,
+ onPending,
+}: FunnyEffectsProps) => {
+ const { t } = useTranslation('rooms', { keyPrefix: 'effects' })
+
+ const getOptions = () => {
+ const processor = videoTrack?.getProcessor() as FaceLandmarksProcessor
+ if (!processor || processor.type != ProcessorType.FACE_LANDMARKS) {
+ return {
+ showGlasses: false,
+ showFrench: false,
+ }
+ }
+ return processor.serialize().options
+ }
+
+ const options = getOptions()
+
+ const toggleFaceLandmarkEffect = async (
+ showEffect: 'showGlasses' | 'showFrench'
+ ) => {
+ const options = getOptions()
+ const processor = videoTrack?.getProcessor() as FaceLandmarksProcessor
+
+ const newOptions = {
+ ...options,
+ [showEffect]: !options[showEffect],
+ }
+
+ onPending(true)
+
+ try {
+ if (!newOptions.showGlasses && !newOptions.showFrench) {
+ await videoTrack.stopProcessor()
+ } else if (options.showGlasses || options.showFrench) {
+ await processor?.update(newOptions)
+ } else {
+ const newProcessor = new FaceLandmarksProcessor(newOptions)
+ await videoTrack.setProcessor(newProcessor)
+ }
+ } catch (e) {
+ console.error('could not update processor', e)
+ } finally {
+ onPending(false)
+ }
+ }
+
+ const getLabelAction = (enabled: boolean) => (enabled ? 'clear' : 'apply')
+
+ return (
+ <div
+ className={css({
+ marginBottom: '1.5rem',
+ })}
+ >
+ <H
+ lvl={3}
+ style={{
+ marginBottom: '1rem',
+ }}
+ variant="bodyXsBold"
+ >
+ {t('faceLandmarks.title')}
+ </H>
+ <div
+ className={css({
+ display: 'flex',
+ gap: '1.25rem',
+ })}
+ >
+ <ToggleButton
+ variant="bigSquare"
+ aria-label={t(
+ `faceLandmarks.glasses.${getLabelAction(options.showGlasses)}`
+ )}
+ tooltip={t(
+ `faceLandmarks.glasses.${getLabelAction(options.showGlasses)}`
+ )}
+ isDisabled={isPending}
+ onChange={async () => await toggleFaceLandmarkEffect('showGlasses')}
+ isSelected={options.showGlasses}
+ data-attr="toggle-glasses"
+ >
+ <RiGlassesLine />
+ </ToggleButton>
+ <ToggleButton
+ variant="bigSquare"
+ aria-label={t(
+ `faceLandmarks.french.${getLabelAction(options.showFrench)}`
+ )}
+ tooltip={t(
+ `faceLandmarks.french.${getLabelAction(options.showFrench)}`
+ )}
+ isDisabled={isPending}
+ onChange={async () => await toggleFaceLandmarkEffect('showFrench')}
+ isSelected={options.showFrench}
+ data-attr="toggle-french"
+ >
+ <RiGoblet2Fill />
+ </ToggleButton>
+ </div>
+ </div>
+ )
+}
diff --git a/src/frontend/src/features/rooms/livekit/hooks/useKonami.ts b/src/frontend/src/features/rooms/livekit/hooks/useKonami.ts
new file mode 100644
index 00000000..e69de29b