Open source video conferencing app powered by LiveKit. Built with Django and React.
✨(front) add blurring feature on join
Nathan Vasse206e74c64568
diff --git a/src/frontend/src/features/home/routes/Home.tsx b/src/frontend/src/features/home/routes/Home.tsxindex a4903d59..113affc8 100644--- a/src/frontend/src/features/home/routes/Home.tsx+++ b/src/frontend/src/features/home/routes/Home.tsx@@ -9,7 +9,6 @@ import { useUser, UserAware } from '@/features/auth'import { JoinMeetingDialog } from '../components/JoinMeetingDialog'import { ProConnectButton } from '@/components/ProConnectButton'import { useCreateRoom } from '@/features/rooms'-import { usePersistentUserChoices } from '@livekit/components-react'import { RiAddLine, RiLink } from '@remixicon/react'import { LaterMeetingDialog } from '@/features/home/components/LaterMeetingDialog'import { IntroSlider } from '@/features/home/components/IntroSlider'@@ -18,6 +17,7 @@ import { ReactNode, useState } from 'react'import { css } from '@/styled-system/css'import { menuRecipe } from '@/primitives/menuRecipe.ts'+import { usePersistentUserChoices } from '@/features/rooms/livekit/hooks/usePersistentUserChoices'const Columns = ({ children }: { children?: ReactNode }) => {return (diff --git a/src/frontend/src/features/rooms/components/Conference.tsx b/src/frontend/src/features/rooms/components/Conference.tsxindex 47638edb..57c28c44 100644--- a/src/frontend/src/features/rooms/components/Conference.tsx+++ b/src/frontend/src/features/rooms/components/Conference.tsx@@ -1,7 +1,7 @@import { useEffect, useMemo, useState } from 'react'import { useQuery } from '@tanstack/react-query'import { useTranslation } from 'react-i18next'-import { LiveKitRoom, type LocalUserChoices } from '@livekit/components-react'+import { LiveKitRoom } from '@livekit/components-react'import { Room, RoomOptions } from 'livekit-client'import { keys } from '@/api/queryKeys'import { queryClient } from '@/api/queryClient'@@ -12,10 +12,11 @@ import { fetchRoom } from '../api/fetchRoom'import { ApiRoom } from '../api/ApiRoom'import { useCreateRoom } from '../api/createRoom'import { InviteDialog } from './InviteDialog'-import { VideoConference } from '../livekit/prefabs/VideoConference'import posthog from 'posthog-js'import { css } from '@/styled-system/css'+import { LocalUserChoices } from '../routes/Room'+import { BackgroundBlurFactory } from '../livekit/components/blur'export const Conference = ({roomId,@@ -111,7 +112,13 @@ export const Conference = ({token={data?.livekit?.token}connect={true}audio={userConfig.audioEnabled}- video={userConfig.videoEnabled}+ video={+ userConfig.videoEnabled && {+ processor: BackgroundBlurFactory.deserializeProcessor(+ userConfig.processorSerialized+ ),+ }+ }connectOptions={connectOptions}className={css({backgroundColor: 'primaryDark.50 !important',diff --git a/src/frontend/src/features/rooms/components/Join.tsx b/src/frontend/src/features/rooms/components/Join.tsxindex 674212da..e537f6aa 100644--- a/src/frontend/src/features/rooms/components/Join.tsx+++ b/src/frontend/src/features/rooms/components/Join.tsx@@ -1,9 +1,5 @@import { useTranslation } from 'react-i18next'-import {- usePersistentUserChoices,- usePreviewTracks,- type LocalUserChoices,-} from '@livekit/components-react'+import { usePreviewTracks } from '@livekit/components-react'import { css } from '@/styled-system/css'import { Screen } from '@/layout/Screen'import { useMemo, useEffect, useRef, useState } from 'react'@@ -13,6 +9,13 @@ import { SelectToggleDevice } from '../livekit/components/controls/SelectToggleDimport { Field } from '@/primitives/Field'import { Form } from '@/primitives'import { HStack, VStack } from '@/styled-system/jsx'+import { Button, Dialog } from '@/primitives'+import { LocalUserChoices } from '../routes/Room'+import { Heading } from 'react-aria-components'+import { RiImageCircleAiFill } from '@remixicon/react'+import { EffectsConfiguration } from '../livekit/components/effects/EffectsConfiguration'+import { usePersistentUserChoices } from '../livekit/hooks/usePersistentUserChoices'+import { BackgroundBlurFactory } from '../livekit/components/blur'const onError = (e: Error) => console.error('ERROR', e)@@ -28,6 +31,7 @@ export const Join = ({saveAudioInputDeviceId,saveVideoInputDeviceId,saveUsername,+ saveProcessorSerialized,} = usePersistentUserChoices({})const [audioDeviceId, setAudioDeviceId] = useState<string>(@@ -37,6 +41,11 @@ export const Join = ({initialUserChoices.videoDeviceId)const [username, setUsername] = useState<string>(initialUserChoices.username)+ const [processor, setProcessor] = useState(+ BackgroundBlurFactory.deserializeProcessor(+ initialUserChoices.processorSerialized+ )+ )useEffect(() => {saveAudioInputDeviceId(audioDeviceId)@@ -49,6 +58,9 @@ export const Join = ({useEffect(() => {saveUsername(username)}, [username, saveUsername])+ useEffect(() => {+ saveProcessorSerialized(processor?.serialize())+ }, [processor, saveProcessorSerialized])const [audioEnabled, setAudioEnabled] = useState(true)const [videoEnabled, setVideoEnabled] = useState(true)@@ -107,11 +119,52 @@ export const Join = ({audioDeviceId,videoDeviceId,username,+ processorSerialized: processor?.serialize(),})}+ const [isEffectsOpen, setEffectsOpen] = useState(false)++ const openEffects = () => {+ setEffectsOpen(true)+ }++ // This hook is used to setup the persisted user choice processor on initialization.+ // So it's on purpose that processor is not included in the deps.+ // We just want to wait for the videoTrack to be loaded to apply the default processor.+ useEffect(() => {+ if (processor && videoTrack && !videoTrack.getProcessor()) {+ videoTrack.setProcessor(processor)+ }+ // eslint-disable-next-line react-hooks/exhaustive-deps+ }, [videoTrack])+return (<Screen footer={false}>+ <Dialog+ isOpen={isEffectsOpen}+ onOpenChange={setEffectsOpen}+ role="dialog"+ type="flex"+ size="large"+ >+ <Heading+ slot="title"+ level={1}+ className={css({+ textStyle: 'h1',+ marginBottom: '1.5rem',+ })}+ >+ {t('effects.title')}+ </Heading>+ <EffectsConfiguration+ videoTrack={videoTrack}+ onSubmit={(processor) => {+ setProcessor(processor)+ }}+ />+ </Dialog><divclassName={css({display: 'flex',@@ -204,8 +257,35 @@ export const Join = ({</p></div>)}+ <div+ className={css({+ position: 'absolute',+ left: 0,+ right: 0,+ bottom: 0,+ height: '20%',+ backgroundImage:+ 'linear-gradient(0deg, rgba(0,0,0,0.8) 0%, rgba(255,255,255,0) 100%)',+ })}+ ></div>+ <div+ className={css({+ position: 'absolute',+ right: 0,+ bottom: '0',+ padding: '1rem',+ })}+ >+ <Button+ variant="whiteCircle"+ onPress={openEffects}+ tooltip={t('effects.description')}+ aria-label={t('effects.description')}+ >+ <RiImageCircleAiFill size={24} />+ </Button>+ </div></div>-<HStack justify="center" padding={1.5}><SelectToggleDevicesource={Track.Source.Microphone}diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.tsindex 2c907e17..781e0d66 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts@@ -11,7 +11,11 @@ import {TIMEOUT_TICK,timerWorkerScript,} from './TimerWorker'-import { BackgroundBlurProcessorInterface, BackgroundOptions } from '.'+import {+ BackgroundBlurProcessorInterface,+ BackgroundOptions,+ ProcessorType,+} from '.'const PROCESSING_WIDTH = 256const PROCESSING_HEIGHT = 144@@ -282,4 +286,11 @@ export class BackgroundBlurCustomProcessorclone() {return new BackgroundBlurCustomProcessor(this.options)}++ serialize() {+ return {+ type: ProcessorType.BLUR,+ options: this.options,+ }+ }}diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.tsindex a2420e7c..5481a9ff 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts@@ -4,7 +4,11 @@ import {ProcessorWrapper,} from '@livekit/track-processors'import { ProcessorOptions, Track } from 'livekit-client'-import { BackgroundBlurProcessorInterface, BackgroundOptions } from '.'+import {+ BackgroundBlurProcessorInterface,+ BackgroundOptions,+ ProcessorType,+} from '.'/*** This is simply a wrapper around track-processor-js Processor@@ -54,4 +58,11 @@ export class BackgroundBlurTrackProcessorJsWrapperblurRadius: this.options!.blurRadius,})}++ serialize() {+ return {+ type: ProcessorType.BLUR,+ options: this.options,+ }+ }}diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.tsindex eae30b65..eaa4a372 100644--- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts+++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts@@ -7,11 +7,21 @@ export type BackgroundOptions = {blurRadius?: number}+export interface ProcessorSerialized {+ type: ProcessorType+ options: BackgroundOptions+}+export interface BackgroundBlurProcessorInterfaceextends TrackProcessor<Track.Kind> {update(opts: BackgroundOptions): voidoptions: BackgroundOptionsclone(): BackgroundBlurProcessorInterface+ serialize(): ProcessorSerialized+}++export enum ProcessorType {+ BLUR = 'blur',}export class BackgroundBlurFactory {@@ -21,13 +31,22 @@ export class BackgroundBlurFactory {)}- static getProcessor(opts: BackgroundOptions) {+ static getProcessor(+ opts: BackgroundOptions+ ): BackgroundBlurProcessorInterface | undefined {if (ProcessorWrapper.isSupported) {return new BackgroundBlurTrackProcessorJsWrapper(opts)}if (BackgroundBlurCustomProcessor.isSupported) {return new BackgroundBlurCustomProcessor(opts)}- return null+ return undefined+ }++ static deserializeProcessor(data?: ProcessorSerialized) {+ if (data?.type === ProcessorType.BLUR) {+ return BackgroundBlurFactory.getProcessor(data?.options)+ }+ return undefined}}diff --git a/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx b/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsxindex 84fbaffa..efb6a91a 100644--- a/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx+++ b/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx@@ -13,7 +13,12 @@ import {RiVideoOffLine,RiVideoOnLine,} from '@remixicon/react'-import { LocalAudioTrack, LocalVideoTrack, Track } from 'livekit-client'+import {+ LocalAudioTrack,+ LocalVideoTrack,+ Track,+ VideoCaptureOptions,+} from 'livekit-client'import { Shortcut } from '@/features/shortcuts/types'@@ -21,6 +26,8 @@ import { ToggleDevice } from '@/features/rooms/livekit/components/controls/Togglimport { css } from '@/styled-system/css'import { ButtonRecipeProps } from '@/primitives/buttonRecipe'import { useEffect } from 'react'+import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices'+import { BackgroundBlurFactory } from '../blur'export type ToggleSource = Exclude<Track.Source,@@ -92,6 +99,39 @@ export const SelectToggleDevice = <T extends ToggleSource>({const { t } = useTranslation('rooms', { keyPrefix: 'join' })const trackProps = useTrackToggle(props)+ const { userChoices } = usePersistentUserChoices({})++ const toggle = () => {+ if (props.source === Track.Source.Camera) {+ /**+ * We need to make sure that we apply the in-memory processor when re-enabling the camera.+ * Before, we had the following bug:+ * 1 - Configure a processor on join screen+ * 2 - Turn off camera on join screen+ * 3 - Join the room+ * 4 - Turn on the camera+ * 5 - No processor is applied to the camera+ * Expected: The processor is applied.+ *+ * See https://github.com/numerique-gouv/meet/pull/309#issuecomment-2622404121+ */+ const processor = BackgroundBlurFactory.deserializeProcessor(+ userChoices.processorSerialized+ )++ const toggle = trackProps.toggle as (+ forceState: boolean,+ captureOptions: VideoCaptureOptions+ ) => Promise<void>++ toggle(!trackProps.enabled, {+ processor: processor,+ } as VideoCaptureOptions)+ } else {+ trackProps.toggle()+ }+ }+const { devices, activeDeviceId, setActiveMediaDevice } =useMediaDeviceSelect({ kind: config.kind, track })@@ -120,6 +160,7 @@ export const SelectToggleDevice = <T extends ToggleSource>({{...trackProps}config={config}variant={variant}+ toggle={toggle}toggleButtonProps={{...(hideMenu? {diff --git a/src/frontend/src/features/rooms/livekit/components/effects/Effects.tsx b/src/frontend/src/features/rooms/livekit/components/effects/Effects.tsxindex 3c2aee55..d8eb1278 100644--- a/src/frontend/src/features/rooms/livekit/components/effects/Effects.tsx+++ b/src/frontend/src/features/rooms/livekit/components/effects/Effects.tsx@@ -2,17 +2,26 @@ import { useLocalParticipant } from '@livekit/components-react'import { LocalVideoTrack } from 'livekit-client'import { css } from '@/styled-system/css'import { EffectsConfiguration } from './EffectsConfiguration'+import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices'export const Effects = () => {const { cameraTrack } = useLocalParticipant()const localCameraTrack = cameraTrack?.track as LocalVideoTrack+ const { saveProcessorSerialized } = usePersistentUserChoices()+return (<divclassName={css({padding: '0 1.5rem',})}>- <EffectsConfiguration videoTrack={localCameraTrack} layout="vertical" />+ <EffectsConfiguration+ videoTrack={localCameraTrack}+ layout="vertical"+ onSubmit={(processor) =>+ saveProcessorSerialized(processor?.serialize())+ }+ /></div>)}diff --git a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsxindex 235a31b8..83cb9bb0 100644--- a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx+++ b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx@@ -1,4 +1,4 @@-import { LocalVideoTrack, Track, TrackProcessor } from 'livekit-client'+import { LocalVideoTrack } from 'livekit-client'import { useEffect, useRef, useState } from 'react'import { useTranslation } from 'react-i18next'import {@@ -33,7 +33,7 @@ export const EffectsConfiguration = ({layout = 'horizontal',}: {videoTrack: LocalVideoTrack- onSubmit?: (processor?: TrackProcessor<Track.Kind.Video>) => void+ onSubmit?: (processor?: BackgroundBlurProcessorInterface) => voidlayout?: 'vertical' | 'horizontal'}) => {const videoRef = useRef<HTMLVideoElement>(null)@@ -68,6 +68,8 @@ export const EffectsConfiguration = ({onSubmit?.(newProcessor)} else {processor?.update({ blurRadius })+ // We want to trigger onSubmit when options changes so the parent component is aware of it.+ onSubmit?.(processor)}} catch (error) {console.error('Error applying blur:', error)diff --git a/src/frontend/src/features/rooms/livekit/hooks/usePersistentUserChoices.ts b/src/frontend/src/features/rooms/livekit/hooks/usePersistentUserChoices.tsnew file mode 100644index 00000000..7e3846fb--- /dev/null+++ b/src/frontend/src/features/rooms/livekit/hooks/usePersistentUserChoices.ts@@ -0,0 +1,56 @@+import { UsePersistentUserChoicesOptions } from '@livekit/components-react'+import React from 'react'+import { LocalUserChoices } from '../../routes/Room'+import { saveUserChoices, loadUserChoices } from '@livekit/components-core'+import { ProcessorSerialized } from '../components/blur'++/**+ * From @livekit/component-react+ *+ * A hook that provides access to user choices stored in local storage, such as+ * selected media devices and their current state (on or off), as well as the user name.+ * @alpha+ */+export function usePersistentUserChoices(+ options: UsePersistentUserChoicesOptions = {}+) {+ const [userChoices, setSettings] = React.useState<LocalUserChoices>(+ loadUserChoices(options.defaults, options.preventLoad ?? false)+ )++ const saveAudioInputEnabled = React.useCallback((isEnabled: boolean) => {+ setSettings((prev) => ({ ...prev, audioEnabled: isEnabled }))+ }, [])+ const saveVideoInputEnabled = React.useCallback((isEnabled: boolean) => {+ setSettings((prev) => ({ ...prev, videoEnabled: isEnabled }))+ }, [])+ const saveAudioInputDeviceId = React.useCallback((deviceId: string) => {+ setSettings((prev) => ({ ...prev, audioDeviceId: deviceId }))+ }, [])+ const saveVideoInputDeviceId = React.useCallback((deviceId: string) => {+ setSettings((prev) => ({ ...prev, videoDeviceId: deviceId }))+ }, [])+ const saveUsername = React.useCallback((username: string) => {+ setSettings((prev) => ({ ...prev, username: username }))+ }, [])+ const saveProcessorSerialized = React.useCallback(+ (processorSerialized?: ProcessorSerialized) => {+ setSettings((prev) => ({ ...prev, processorSerialized }))+ },+ []+ )++ React.useEffect(() => {+ saveUserChoices(userChoices, options.preventSave ?? false)+ }, [userChoices, options.preventSave])++ return {+ userChoices,+ saveAudioInputEnabled,+ saveVideoInputEnabled,+ saveAudioInputDeviceId,+ saveVideoInputDeviceId,+ saveUsername,+ saveProcessorSerialized,+ }+}diff --git a/src/frontend/src/features/rooms/livekit/prefabs/ControlBar/ControlBar.tsx b/src/frontend/src/features/rooms/livekit/prefabs/ControlBar/ControlBar.tsxindex c61161de..397ec3e4 100644--- a/src/frontend/src/features/rooms/livekit/prefabs/ControlBar/ControlBar.tsx+++ b/src/frontend/src/features/rooms/livekit/prefabs/ControlBar/ControlBar.tsx@@ -1,11 +1,11 @@import { Track } from 'livekit-client'import * as React from 'react'-import { usePersistentUserChoices } from '@livekit/components-react'import { MobileControlBar } from './MobileControlBar'import { DesktopControlBar } from './DesktopControlBar'import { SettingsDialogProvider } from '../../components/controls/SettingsDialogContext'import { useIsMobile } from '@/utils/useIsMobile'+import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices'/** @public */export type ControlBarControls = {diff --git a/src/frontend/src/features/rooms/routes/Room.tsx b/src/frontend/src/features/rooms/routes/Room.tsxindex 2424495e..c0c2cd7a 100644--- a/src/frontend/src/features/rooms/routes/Room.tsx+++ b/src/frontend/src/features/rooms/routes/Room.tsx@@ -1,7 +1,7 @@import { useEffect, useState } from 'react'import {usePersistentUserChoices,- type LocalUserChoices,+ type LocalUserChoices as LocalUserChoicesLK,} from '@livekit/components-react'import { useParams } from 'wouter'import { ErrorScreen } from '@/components/ErrorScreen'@@ -9,6 +9,11 @@ import { useUser, UserAware } from '@/features/auth'import { Conference } from '../components/Conference'import { Join } from '../components/Join'import { useKeyboardShortcuts } from '@/features/shortcuts/useKeyboardShortcuts'+import { ProcessorSerialized } from '../livekit/components/blur'++export type LocalUserChoices = LocalUserChoicesLK & {+ processorSerialized?: ProcessorSerialized+}export const Room = () => {const { isLoggedIn } = useUser()diff --git a/src/frontend/src/features/settings/components/tabs/AccountTab.tsx b/src/frontend/src/features/settings/components/tabs/AccountTab.tsxindex 89fb831c..77c9f655 100644--- a/src/frontend/src/features/settings/components/tabs/AccountTab.tsx+++ b/src/frontend/src/features/settings/components/tabs/AccountTab.tsx@@ -1,15 +1,13 @@import { A, Badge, Button, DialogProps, Field, H, P } from '@/primitives'import { Trans, useTranslation } from 'react-i18next'-import {- usePersistentUserChoices,- useRoomContext,-} from '@livekit/components-react'+import { useRoomContext } from '@livekit/components-react'import { logoutUrl, useUser } from '@/features/auth'import { css } from '@/styled-system/css'import { TabPanel, TabPanelProps } from '@/primitives/Tabs'import { HStack } from '@/styled-system/jsx'import { useState } from 'react'import { ProConnectButton } from '@/components/ProConnectButton'+import { usePersistentUserChoices } from '@/features/rooms/livekit/hooks/usePersistentUserChoices'export type AccountTabProps = Pick<DialogProps, 'onOpenChange'> &Pick<TabPanelProps, 'id'>diff --git a/src/frontend/src/locales/en/rooms.json b/src/frontend/src/locales/en/rooms.jsonindex 49b1bdff..b58ac8a3 100644--- a/src/frontend/src/locales/en/rooms.json+++ b/src/frontend/src/locales/en/rooms.json@@ -18,6 +18,10 @@"enable": "Enable microphone","label": "Microphone"},+ "effects": {+ "description": "Apply effects",+ "title": "Effects"+ },"heading": "Join the meeting","joinLabel": "Join","joinMeeting": "Join meeting",diff --git a/src/frontend/src/locales/fr/rooms.json b/src/frontend/src/locales/fr/rooms.jsonindex 1924b7eb..d99cae54 100644--- a/src/frontend/src/locales/fr/rooms.json+++ b/src/frontend/src/locales/fr/rooms.json@@ -19,6 +19,10 @@"label": "Microphone"},"heading": "Rejoindre la réunion",+ "effects": {+ "description": "Appliquer des effets",+ "title": "Effets"+ },"joinLabel": "Rejoindre","joinMeeting": "Rejoindre la réjoindre","toggleOff": "Cliquez pour désactiver",