Skip to content

armgit / meet

Public Imported from GitHub
0

Open source video conferencing app powered by LiveKit. Built with Django and React.

🩹(frontend) unblock virtual background loading under bearer auth

lebaudantoinec2007929f461
diff --git a/src/frontend/src/features/files/hooks/useResolvedMediaUrls.ts b/src/frontend/src/features/files/hooks/useResolvedMediaUrls.ts
new file mode 100644
index 00000000..538635f6
--- /dev/null
+++ b/src/frontend/src/features/files/hooks/useResolvedMediaUrls.ts
@@ -0,0 +1,56 @@
+import { useCallback, useEffect, useState } from 'react'
+import { useSnapshot } from 'valtio'
+import { accessTokenStore } from '@/stores/accessToken'
+import { resolveMediaUrl } from '../utils/resolveMediaUrl'
+
+/**
+ * Reactive companion of resolveMediaUrl for browser-native consumers
+ * (CSS url(), img src attributes): resolves a list of /media/ URLs and
+ * returns a stable lookup, identity in regular mode.
+ *
+ * Object URLs come from the shared session-lifetime cache and are never
+ * revoked here: they may be used concurrently by the background
+ * processors.
+ */
+export const useResolvedMediaUrls = (
+ urls: (string | null | undefined)[]
+): ((url: string) => string) => {
+ const [resolved, setResolved] = useState<Record<string, string>>({})
+ const { accessToken } = useSnapshot(accessTokenStore)
+
+ // Stable dependency for the effect, insensitive to array identity
+ const urlsKey = urls.filter(Boolean).sort().join('\n')
+
+ useEffect(() => {
+ if (!accessToken || !urlsKey) {
+ return
+ }
+
+ let isMounted = true
+
+ const resolveAll = async () => {
+ const entries = await Promise.all(
+ urlsKey.split('\n').map(async (url) => {
+ try {
+ return [url, await resolveMediaUrl(url)] as const
+ } catch (error) {
+ console.warn(error)
+ return [url, url] as const
+ }
+ })
+ )
+ if (isMounted) {
+ setResolved(Object.fromEntries(entries))
+ }
+ }
+ resolveAll()
+
+ return () => {
+ isMounted = false
+ }
+ }, [accessToken, urlsKey])
+
+ // Stable identity so that consumers can safely list the resolver in
+ // their memo dependencies: it only changes when resolutions land.
+ return useCallback((url: string) => resolved[url] ?? url, [resolved])
+}
diff --git a/src/frontend/src/features/files/utils/resolveMediaUrl.ts b/src/frontend/src/features/files/utils/resolveMediaUrl.ts
new file mode 100644
index 00000000..07d53140
--- /dev/null
+++ b/src/frontend/src/features/files/utils/resolveMediaUrl.ts
@@ -0,0 +1,68 @@
+import { getAccessToken } from '@/stores/accessToken'
+
+// Session-lifetime cache: object URLs are shared between every consumer
+// of a given media (background processors, thumbnails) and are therefore
+// never revoked - their number is bounded by the user's custom
+// backgrounds, and they die with the page like the access token does.
+const objectUrlCache = new Map<string, string>()
+
+// Upper bound on a single media fetch, so a stalled request rejects
+// instead of leaving the background processor awaiting forever.
+const MEDIA_FETCH_TIMEOUT_MS = 15000 // 15 seconds
+
+/**
+ * Resolve an authenticated /media/ URL for the embedded (token) mode.
+ *
+ * Media files are served behind an nginx auth_request subrequest that
+ * authenticates the original request. In regular mode the session cookie
+ * rides along browser-native loads (img.src, CSS url()) and the URL is
+ * returned unchanged, without any fetch. In embedded mode the
+ * third-party cookie is blocked and native loads cannot carry the
+ * Authorization header, so the media is fetched here with the Bearer
+ * header - which the media-auth endpoint accepts, as it sits behind the
+ * default authentication stack - and exposed as a blob object URL.
+ */
+export const resolveMediaUrl = async (url: string): Promise<string> => {
+ const accessToken = getAccessToken()
+
+ if (!accessToken) {
+ return url
+ }
+
+ const cached = objectUrlCache.get(url)
+ if (cached) {
+ return cached
+ }
+
+ const response = await fetch(url, {
+ headers: { Authorization: `Bearer ${accessToken}` },
+ signal: AbortSignal.timeout(MEDIA_FETCH_TIMEOUT_MS),
+ })
+
+ if (!response.ok) {
+ throw new Error(
+ `Failed to resolve media url ${url}: HTTP ${response.status}`
+ )
+ }
+
+ const objectUrl = URL.createObjectURL(await response.blob())
+ objectUrlCache.set(url, objectUrl)
+
+ return objectUrl
+}
+
+/**
+ * Non-throwing variant of resolveMediaUrl for callers that must not fail
+ * (e.g. background processor init/update). Any error - network, DNS,
+ * timeout, partial download - is logged and the raw URL is returned.
+ */
+export const resolveMediaUrlOrFallback = async (
+ url: string
+): Promise<string> => {
+ try {
+ return await resolveMediaUrl(url)
+ } catch (error) {
+ console.warn(error)
+ return url
+ }
+}
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts
index 73896e7a..8474967a 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts
@@ -1,4 +1,5 @@
import type { ProcessorOptions, Track } from 'livekit-client'
+import { resolveMediaUrlOrFallback } from '@/features/files/utils/resolveMediaUrl'
import {
FilesetResolver,
ImageSegmenter,
@@ -85,7 +86,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
this.sourceSettings = this.source!.getSettings()
this.videoElement = opts.element as HTMLVideoElement
- this._initVirtualBackgroundImage()
+ await this._initVirtualBackgroundImage()
this._createMainCanvas()
this._createMaskCanvas()
@@ -103,7 +104,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
captureEvent('firefox-blurring-init', {})
}
- _initVirtualBackgroundImage() {
+ async _initVirtualBackgroundImage() {
if (this.options.type !== 'virtual') {
return
}
@@ -113,15 +114,20 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
this.virtualBackgroundImage &&
this.virtualBackgroundImage.src !== this.options.imagePath
if (this.options.imagePath || needsUpdate) {
+ // Embedded (token) mode: img.src cannot carry the Authorization
+ // header, resolve the media to a blob object URL first. Identity
+ // in regular mode. Falls back to the raw URL on failure rather
+ // than rejecting the processor init/update.
+ const imagePath = await resolveMediaUrlOrFallback(this.options.imagePath!)
this.virtualBackgroundImage = document.createElement('img')
this.virtualBackgroundImage.crossOrigin = 'anonymous'
- this.virtualBackgroundImage.src = this.options.imagePath!
+ this.virtualBackgroundImage.src = imagePath
}
}
async update(opts: ProcessorConfig): Promise<void> {
this.options = opts
- this._initVirtualBackgroundImage()
+ await this._initVirtualBackgroundImage()
}
_initWorker() {
diff --git a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts
index 5087215f..cdbaf006 100644
--- a/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts
+++ b/src/frontend/src/features/rooms/livekit/components/blur/UnifiedBackgroundTrackProcessor.ts
@@ -1,4 +1,5 @@
import type { ProcessorOptions, Track } from 'livekit-client'
+import { resolveMediaUrlOrFallback } from '@/features/files/utils/resolveMediaUrl'
import {
ProcessorWrapper,
BackgroundProcessor,
@@ -47,7 +48,16 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInter
}
async init(opts: ProcessorOptions<Track.Kind>) {
- return this.processor.init(opts)
+ await this.processor.init(opts)
+ // Embedded (token) mode: the constructor passed the raw imagePath,
+ // whose native load cannot carry the Authorization header. Swap it
+ // for a resolved blob object URL. No-op in regular mode.
+ if (this.opts.type === 'virtual') {
+ const imagePath = await resolveMediaUrlOrFallback(this.opts.imagePath)
+ if (imagePath !== this.opts.imagePath) {
+ await this.processor.updateTransformerOptions({ imagePath })
+ }
+ }
}
async restart(opts: ProcessorOptions<Track.Kind>) {
@@ -59,6 +69,12 @@ export class UnifiedBackgroundTrackProcessor implements BackgroundProcessorInter
}
async update(opts: ProcessorConfig): Promise<void> {
+ if (opts.type === 'virtual') {
+ opts = {
+ ...opts,
+ imagePath: await resolveMediaUrlOrFallback(opts.imagePath),
+ }
+ }
this.opts = opts
const newProcessorType =
diff --git a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx
index 1c2b486a..542250a0 100644
--- a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx
+++ b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx
@@ -8,6 +8,7 @@ import {
ProcessorType,
} from '../blur'
import { css } from '@/styled-system/css'
+import { useResolvedMediaUrls } from '@/features/files/hooks/useResolvedMediaUrls'
import { Button, Dialog, H, P, Text, ToggleButton } from '@/primitives'
import { VisualOnlyTooltip } from '@/primitives/VisualOnlyTooltip'
import { HStack, styled } from '@/styled-system/jsx'
@@ -280,6 +281,14 @@ export const EffectsConfiguration = ({
filesQ.data.count >= appConfig.background_image.max_count_by_user) ??
false
+ // Thumbnails are browser-native loads (CSS url()) which cannot carry
+ // the Authorization header in embedded (token) mode: resolve them. The
+ // processor configs keep the stable raw URLs - they are persisted in
+ // the user choices - and the processors resolve them internally.
+ const resolveMediaUrl = useResolvedMediaUrls(
+ (filesQ.data?.results ?? []).map((file) => file.url)
+ )
+
const getHandleSelectChangeFile = useCallback(
(file: ApiFileItem) => {
return async () => {
@@ -757,7 +766,7 @@ export const EffectsConfiguration = ({
bgSize: 'cover',
})}
style={{
- backgroundImage: `url(${option.file.url!})`,
+ backgroundImage: `url(${resolveMediaUrl(option.file.url!)})`,
}}
data-attr={`toggle-virtual-${option.file.id}`}
/>
diff --git a/src/frontend/src/stores/userChoices.ts b/src/frontend/src/stores/userChoices.ts
index 04ed51d6..0b3647bd 100644
--- a/src/frontend/src/stores/userChoices.ts
+++ b/src/frontend/src/stores/userChoices.ts
@@ -1,4 +1,6 @@
import { proxy, subscribe } from 'valtio'
+import { initializeAccessTokenFromFragment } from '@/features/auth/api/exchangeAccessToken'
+import { getAccessToken } from '@/stores/accessToken'
import {
ProcessorConfig,
ProcessorType,
@@ -57,10 +59,19 @@ if (userChoicesStore.processorConfig?.type === ProcessorType.VIRTUAL) {
// we restore clear the processor config to avoid displaying a black screen.
userChoicesStore.processorConfig = undefined
} else if (userChoicesStore.processorConfig.fileId) {
+ // Embedded (token) mode: this module loads before the transit code
+ // exchange has settled - wait for it, and carry the Bearer header,
+ // otherwise the check below would wrongly clear the config.
+ await initializeAccessTokenFromFragment()
+ const accessToken = getAccessToken()
+
// Checking if the image is still available / accessible
await fetch(userChoicesStore.processorConfig.imagePath, {
// We bypass the cache to ensure we have access
cache: 'reload',
+ ...(accessToken && {
+ headers: { Authorization: `Bearer ${accessToken}` },
+ }),
})
.then((response) => {
// if we cannot fetch the image (likely a 401 from the backend because