Open source video conferencing app powered by LiveKit. Built with Django and React.
✨(frontend) introduce the camera stage
lebaudantoine38c131e02cff
diff --git a/src/frontend/src/features/pip/components/PipView.tsx b/src/frontend/src/features/pip/components/PipView.tsxindex da463acc..c99cb95f 100644--- a/src/frontend/src/features/pip/components/PipView.tsx+++ b/src/frontend/src/features/pip/components/PipView.tsx@@ -1,7 +1,9 @@+import { styled } from '@/styled-system/jsx'import { PipControlBar } from './PipControlBar'import { PipFloatingReactions } from './PipFloatingReactions'+import { PipStage } from './layout/PipStage'import { ReactionsToolbar } from '@/features/reactions/components/toolbar/ReactionsToolbar'-import { styled } from '@/styled-system/jsx'+import { useReactionsToolbar } from '@/features/reactions/hooks/useReactionsToolbar'const Container = styled('div', {base: {@@ -11,6 +13,7 @@ const Container = styled('div', {display: 'grid',gridTemplateRows: 'minmax(0, 1fr) auto auto',backgroundColor: 'primaryDark.50',+ transition: 'padding .5s cubic-bezier(0.4,0,0.2,1) 5ms',// Disable LiveKit's own border-radius on tiles so our containers// (GridCell, Thumbnail, StageFrame) own the clipping exclusively.'--lk-border-radius': '4px',@@ -29,11 +32,24 @@ const Container = styled('div', {width: '100%',},},+ variants: {+ isReactionToolbarOpen: {+ true: {+ paddingBottom:+ 'calc(var(--sizes-room-reaction-toolbar-height) + var(--sizes-room-control-bar) + 1.125rem)',+ },+ false: {+ paddingBottom: 'var(--sizes-room-control-bar)',+ },+ },+ },})export const PipView = () => {+ const { isOpen: isReactionToolbarOpen } = useReactionsToolbar()return (- <Container>+ <Container isReactionToolbarOpen={isReactionToolbarOpen}>+ <PipStage /><ReactionsToolbar adjustedCentering={false} /><PipControlBar showScreenShare={false} /><PipFloatingReactions />diff --git a/src/frontend/src/features/pip/components/layout/PipFocusLayout.tsx b/src/frontend/src/features/pip/components/layout/PipFocusLayout.tsxnew file mode 100644index 00000000..e09b5ef0--- /dev/null+++ b/src/frontend/src/features/pip/components/layout/PipFocusLayout.tsx@@ -0,0 +1,81 @@+import { memo } from 'react'+import type { TrackReferenceOrPlaceholder } from '@livekit/components-core'+import { styled } from '@/styled-system/jsx'+import { ParticipantTile } from '@/features/rooms/livekit/components/ParticipantTile'+import { getTrackKey } from '../../utils/pipTrackSelection'++type PipFocusLayoutProps = {+ mainTrack?: TrackReferenceOrPlaceholder+ thumbnailTrack?: TrackReferenceOrPlaceholder+}++/**+ * Focus layout used when 1-2 tracks are visible in the PiP window.+ *+ * The main tile is letterboxed (object-fit: contain) so the camera is+ * never stretched to a non-video aspect and leaves dark padding+ * above/below when the window shape doesn't match the source.+ * The thumbnail keeps the usual cover fill.+ */+export const PipFocusLayout = memo(+ ({ mainTrack, thumbnailTrack }: PipFocusLayoutProps) => {+ return (+ <FocusContainer>+ {mainTrack && (+ <MainSlot>+ <ParticipantTile+ key={getTrackKey(mainTrack)}+ trackRef={mainTrack}+ />+ </MainSlot>+ )}+ {thumbnailTrack && (+ <Thumbnail>+ <ParticipantTile+ key={getTrackKey(thumbnailTrack)}+ trackRef={thumbnailTrack}+ />+ </Thumbnail>+ )}+ </FocusContainer>+ )+ }+)+PipFocusLayout.displayName = 'PipFocusLayout'++const FocusContainer = styled('div', {+ base: {+ position: 'relative',+ width: '100%',+ height: '100%',+ borderRadius: '4px',+ overflow: 'hidden',+ backgroundColor: 'primaryDark.100',+ },+})++const MainSlot = styled('div', {+ base: {+ width: '100%',+ height: '100%',+ '& .lk-participant-media-video': {+ objectFit: 'contain',+ },+ },+})++const Thumbnail = styled('div', {+ base: {+ position: 'absolute',+ right: '1rem',+ bottom: '1rem',+ width: '42%',+ maxWidth: '220px',+ minWidth: '140px',+ aspectRatio: '16 / 9',+ borderRadius: '4px',+ overflow: 'hidden',+ boxShadow: 'md',+ zIndex: 2,+ },+})diff --git a/src/frontend/src/features/pip/components/layout/PipGridLayout.tsx b/src/frontend/src/features/pip/components/layout/PipGridLayout.tsxnew file mode 100644index 00000000..5221ef85--- /dev/null+++ b/src/frontend/src/features/pip/components/layout/PipGridLayout.tsx@@ -0,0 +1,76 @@+import { memo, useMemo, useRef } from 'react'+import type { TrackReferenceOrPlaceholder } from '@livekit/components-core'+import { styled } from '@/styled-system/jsx'+import { ParticipantTile } from '@/features/rooms/livekit/components/ParticipantTile'+import { usePipElementSize } from '../../hooks/usePipElementSize'+import { computePipGridLayout } from '../../utils/pipGrid'+import { getTrackKey } from '../../utils/pipTrackSelection'++type PipGridLayoutProps = {+ tracks: TrackReferenceOrPlaceholder[]+}++/**+ * Adaptive grid used when 3+ tracks are visible in the PiP window.+ *+ * All grid math (shape choice + partial-row stretching) is delegated to+ * `computePipGridLayout`. This component only measures the container,+ * applies the returned placements, and plays a FLIP animation when the+ * tile set or grid shape changes (participant joins/leaves or shape shift).+ *+ * Tiles keep a stable key so resizing never remounts <video> elements.+ */+export const PipGridLayout = memo(({ tracks }: PipGridLayoutProps) => {+ const containerRef = useRef<HTMLDivElement>(null)+ const { width, height } = usePipElementSize(containerRef)++ const { rows, subColumns, placements } = useMemo(+ () => computePipGridLayout(tracks.length, width, height),+ [tracks.length, width, height]+ )++ const gridStyle = useMemo(+ () => ({+ gridTemplateColumns: `repeat(${subColumns}, minmax(0, 1fr))`,+ gridTemplateRows: `repeat(${rows}, minmax(0, 1fr))`,+ }),+ [subColumns, rows]+ )++ return (+ <GridContainer ref={containerRef} style={gridStyle}>+ {tracks.map((track, index) => (+ <GridCell key={getTrackKey(track)} style={placements[index]}>+ <ParticipantTile trackRef={track} />+ </GridCell>+ ))}+ </GridContainer>+ )+})+PipGridLayout.displayName = 'PipGridLayout'++const GridContainer = styled('div', {+ base: {+ width: '100%',+ height: '100%',+ display: 'grid',+ gap: '0.25rem',+ },+})++const GridCell = styled('div', {+ base: {+ position: 'relative',+ minWidth: 0,+ minHeight: 0,+ borderRadius: '4px',+ overflow: 'hidden',+ backgroundColor: 'primaryDark.100',+ // Paint on own layer so FLIP transforms don't trigger layout thrash.+ willChange: 'transform',+ '& .lk-participant-tile': {+ width: '100%',+ height: '100%',+ },+ },+})diff --git a/src/frontend/src/features/pip/components/layout/PipStage.tsx b/src/frontend/src/features/pip/components/layout/PipStage.tsxnew file mode 100644index 00000000..b59ffe7e--- /dev/null+++ b/src/frontend/src/features/pip/components/layout/PipStage.tsx@@ -0,0 +1,77 @@+import { useMemo } from 'react'+import { useTracks } from '@livekit/components-react'+import { RoomEvent, Track } from 'livekit-client'+import { PipFocusLayout } from './PipFocusLayout'+import { PipGridLayout } from './PipGridLayout'+import { StageFrame } from './StageFrame'+import {+ isTrackReference,+ TrackReferenceOrPlaceholder,+} from '@livekit/components-core'++/**+ * PipStage picks between two layouts based on track count:+ * - Focus mode (≤ 2 tracks): one main track + one thumbnail overlay.+ * - Grid mode (3+ tracks): adaptive tiling.+ */+export const PipStage = () => {+ const tracks = useTracks(+ [+ { source: Track.Source.Camera, withPlaceholder: true },+ { source: Track.Source.ScreenShare, withPlaceholder: false },+ ],+ { updateOnlyOn: [RoomEvent.ActiveSpeakersChanged], onlySubscribed: false }+ )++ const screenShareTrack = useMemo(() => {+ return tracks+ .filter((track) => isTrackReference(track))+ .find((track) => track.publication.source === Track.Source.ScreenShare)+ }, [tracks])++ const cameraTracks = useMemo(+ () =>+ tracks.filter(+ (track: TrackReferenceOrPlaceholder) =>+ track.source === Track.Source.Camera+ ),+ [tracks]+ )++ if (tracks.length === 0) return null++ /**+ * The focus layout shows one main track with one thumbnail overlay,+ * so it can only fit 2 tracks. Beyond that we switch to the grid.+ */+ if (tracks.length > 2) {+ // Grid mode: 3+ tracks. Screen share goes first so it leads the grid.+ const gridTracks = screenShareTrack+ ? [screenShareTrack, ...cameraTracks]+ : cameraTracks+ return (+ <StageFrame>+ <PipGridLayout tracks={gridTracks} />+ </StageFrame>+ )+ }++ const localCameraTrack = cameraTracks.find(+ (track) => track.participant?.isLocal+ )++ const remoteCameraTrack = cameraTracks.find(+ (track) => !track.participant?.isLocal+ )++ const mainTrack = screenShareTrack ?? remoteCameraTrack ?? localCameraTrack++ const thumbnailTrack =+ mainTrack === localCameraTrack ? undefined : localCameraTrack++ return (+ <StageFrame>+ <PipFocusLayout mainTrack={mainTrack} thumbnailTrack={thumbnailTrack} />+ </StageFrame>+ )+}diff --git a/src/frontend/src/features/pip/components/layout/StageFrame.tsx b/src/frontend/src/features/pip/components/layout/StageFrame.tsxnew file mode 100644index 00000000..46e34347--- /dev/null+++ b/src/frontend/src/features/pip/components/layout/StageFrame.tsx@@ -0,0 +1,25 @@+import { useTranslation } from 'react-i18next'+import { styled } from '@/styled-system/jsx'++export const StageFrame = ({ children }: { children: React.ReactNode }) => {+ const { t } = useTranslation('rooms', {+ keyPrefix: 'pictureInPicture',+ })+ return (+ <Container role="region" aria-label={t('stage')} {...{ inert: '' }}>+ {children}+ </Container>+ )+}++const Container = styled('div', {+ base: {+ position: 'relative',+ minWidth: 0,+ minHeight: 0,+ marginLeft: '0.5rem',+ marginRight: '0.5rem',+ borderRadius: '4px',+ overflow: 'hidden',+ },+})diff --git a/src/frontend/src/features/pip/hooks/usePictureInPicture.ts b/src/frontend/src/features/pip/hooks/usePictureInPicture.tsindex 6842d0de..313d21d0 100644--- a/src/frontend/src/features/pip/hooks/usePictureInPicture.ts+++ b/src/frontend/src/features/pip/hooks/usePictureInPicture.ts@@ -1,8 +1,10 @@import { ref, useSnapshot } from 'valtio'import { useCallback, useMemo } from 'react'-import { IS_PIP_SUPPORTED } from '@/features/pip/utils'import { documentPictureInPictureStore } from '@/stores/documentPictureInPicture'+export const IS_PIP_SUPPORTED =+ typeof globalThis !== 'undefined' && 'documentPictureInPicture' in globalThis+export const usePictureInPicture = () => {const { window: pipWindowRef } = useSnapshot(documentPictureInPictureStore)diff --git a/src/frontend/src/features/pip/hooks/usePipElementSize.ts b/src/frontend/src/features/pip/hooks/usePipElementSize.tsnew file mode 100644index 00000000..f37042de--- /dev/null+++ b/src/frontend/src/features/pip/hooks/usePipElementSize.ts@@ -0,0 +1,42 @@+import { useCallback, useEffect, useState, type RefObject } from 'react'++type Size = { width: number; height: number }++/**+ * Observes an element's size, even when mounted in the PiP document.+ * Resolves `ResizeObserver` from the element's own window.+ */+export const usePipElementSize = <T extends HTMLElement>(+ ref: RefObject<T | null>+): Size => {+ const [size, setSize] = useState<Size>({ width: 0, height: 0 })++ const measure = useCallback(() => {+ const el = ref.current+ if (!el) return+ const rect = el.getBoundingClientRect()+ setSize({ width: rect.width, height: rect.height })+ }, [ref])++ useEffect(() => {+ const el = ref.current+ if (!el) return++ measure()++ const RO =+ el.ownerDocument.defaultView?.ResizeObserver ?? globalThis.ResizeObserver+ if (!RO) return++ const observer = new RO((entries) => {+ const entry = entries[0]+ if (!entry) return+ const { width, height } = entry.contentRect+ setSize({ width, height })+ })+ observer.observe(el)+ return () => observer.disconnect()+ }, [ref, measure])++ return size+}diff --git a/src/frontend/src/features/pip/utils.ts b/src/frontend/src/features/pip/utils.tsdeleted file mode 100644index b333fedb..00000000--- a/src/frontend/src/features/pip/utils.ts+++ /dev/null@@ -1,2 +0,0 @@-export const IS_PIP_SUPPORTED =- typeof globalThis !== 'undefined' && 'documentPictureInPicture' in globalThisdiff --git a/src/frontend/src/features/pip/utils/pipGrid.ts b/src/frontend/src/features/pip/utils/pipGrid.tsnew file mode 100644index 00000000..ce73dcdc--- /dev/null+++ b/src/frontend/src/features/pip/utils/pipGrid.ts@@ -0,0 +1,114 @@+export type PipTilePlacement = {+ gridColumn: string+ gridRow: number+}++export type PipGridLayout = {+ cols: number+ rows: number+ /** Number of CSS sub-columns; use as `repeat(subColumns, 1fr)`. */+ subColumns: number+ /** One entry per tile, in input order. */+ placements: PipTilePlacement[]+}++/**+ * Target tile aspect ratio used to score candidate grid shapes.+ *+ * Video sources are 16:9, but picking 16:9 as the target makes the+ * scorer indifferent between a stretched 2-col slab (aspect ~2.7) and a+ * squarer 3-col tile (aspect ~1.2) because log distance is symmetric.+ * The UI works better with square, face-friendly tiles. This target keeps+ * wide windows from collapsing to 2 columns with short, stretched rows+ * and pushes the scorer to add a column instead.+ */+const TARGET_TILE_ASPECT = 1++/**+ * Smallest count from which we force at least two columns.+ * For 1-3 participants it is acceptable to stack vertically in tall+ * windows, but from 4 people onwards we keep >=2 columns to+ * avoid endless vertical scrolling; the scorer handles the rest.+ */+const FORCE_TWO_COLS_COUNT = 4++const pickGridShape = (+ count: number,+ width: number,+ height: number+): { cols: number; rows: number } => {+ if (count <= 1) return { cols: 1, rows: Math.max(1, count) }+ if (width <= 0 || height <= 0) return { cols: count, rows: 1 }++ const minCols = count >= FORCE_TWO_COLS_COUNT ? 2 : 1++ let best = {+ cols: minCols,+ rows: Math.ceil(count / minCols),+ score: -Infinity,+ }+ for (let cols = minCols; cols <= count; cols++) {+ const rows = Math.ceil(count / cols)+ const tileW = width / cols+ const tileH = height / rows+ if (tileW <= 0 || tileH <= 0) continue++ // Score: aspect close to target, few empty cells, large tile area,+ // and a tiny bias toward fewer rows so ties (perfectly square shapes)+ // resolve in favour of a shorter, wider grid.+ const aspectScore = -Math.abs(Math.log(tileW / tileH / TARGET_TILE_ASPECT))+ const emptyCells = cols * rows - count+ const fillScore = -emptyCells * 0.1+ const areaScore = Math.log(tileW * tileH) * 0.5+ const rowsPenalty = -rows * 0.01++ const score = aspectScore * 2 + fillScore + areaScore + rowsPenalty+ if (score > best.score) best = { cols, rows, score }+ }+ return { cols: best.cols, rows: best.rows }+}++/**+ * Pure function. Given a tile count and stage dimensions, returns the CSS+ * grid layout for the PiP stage:+ *+ * - picks a cols x rows shape close to 16:9 tiles,+ * - stretches any partial last row so its tiles share the full row width+ * (no empty cells, no small centered tile).+ *+ * Callers consume the result directly: `subColumns` feeds+ * `grid-template-columns: repeat(N, 1fr)` and each tile reads its own+ * `gridColumn`/`gridRow` from `placements`.+ */+export const computePipGridLayout = (+ count: number,+ width: number,+ height: number+): PipGridLayout => {+ if (count <= 0) {+ return { cols: 1, rows: 1, subColumns: 1, placements: [] }+ }++ const { cols, rows } = pickGridShape(count, width, height)+ const tilesInLastRow = count - cols * (rows - 1)+ const hasPartialRow = tilesInLastRow > 0 && tilesInLastRow < cols++ const subColumns = hasPartialRow ? cols * tilesInLastRow : cols+ const fullRowSpan = hasPartialRow ? tilesInLastRow : 1+ const lastRowSpan = hasPartialRow ? cols : 1++ const placements: PipTilePlacement[] = []+ for (let i = 0; i < count; i++) {+ const row = Math.floor(i / cols)+ const colIndex = i % cols+ const isLastRow = row === rows - 1 && hasPartialRow+ const span = isLastRow ? lastRowSpan : fullRowSpan+ const colStart = colIndex * span + 1+ placements.push({+ gridColumn: `${colStart} / span ${span}`,+ gridRow: row + 1,+ })+ }++ return { cols, rows, subColumns, placements }+}diff --git a/src/frontend/src/features/pip/utils/pipTrackSelection.ts b/src/frontend/src/features/pip/utils/pipTrackSelection.tsnew file mode 100644index 00000000..96062922--- /dev/null+++ b/src/frontend/src/features/pip/utils/pipTrackSelection.ts@@ -0,0 +1,16 @@+import {+ isTrackReference,+ TrackReferenceOrPlaceholder,+} from '@livekit/components-core'++/**+ * Produces a stable React key for a track so resizes/reshuffles of the grid+ * do not remount the underlying <video> element.+ */+export const getTrackKey = (track: TrackReferenceOrPlaceholder): string => {+ const identity = track.participant?.identity ?? 'unknown'+ if (isTrackReference(track)) {+ return `${identity}::${track.source}::${track.publication.trackSid}`+ }+ return `${identity}::${track.source}::placeholder`+}diff --git a/src/frontend/src/locales/de/rooms.json b/src/frontend/src/locales/de/rooms.jsonindex 07eadd20..ea8b6454 100644--- a/src/frontend/src/locales/de/rooms.json+++ b/src/frontend/src/locales/de/rooms.json@@ -237,6 +237,7 @@"description": "Im Bild-im-Bild-Modus bleiben Sie mit dem Anruf verbunden, während Sie andere Aufgaben erledigen.","bringBack": "Anruf hierher zurückholen"},+ "stage": "Teilnehmer","controlBar": "Besprechungssteuerung"},"options": {diff --git a/src/frontend/src/locales/en/rooms.json b/src/frontend/src/locales/en/rooms.jsonindex 1c724203..17afa83e 100644--- a/src/frontend/src/locales/en/rooms.json+++ b/src/frontend/src/locales/en/rooms.json@@ -237,6 +237,7 @@"description": "Picture-in-Picture mode allows you to stay connected to the call while performing other tasks.","bringBack": "Bring the call back here"},+ "stage": "Participants","controlBar": "Meeting controls"},"options": {diff --git a/src/frontend/src/locales/fr/rooms.json b/src/frontend/src/locales/fr/rooms.jsonindex 0f03f877..f408c50f 100644--- a/src/frontend/src/locales/fr/rooms.json+++ b/src/frontend/src/locales/fr/rooms.json@@ -237,6 +237,7 @@"description": "Le mode image dans l'image vous permet de rester connecté à l'appel tout en effectuant d'autres tâches.","bringBack": "Ramener l'appel ici"},+ "stage": "Participants","controlBar": "Commandes de la réunion"},"options": {diff --git a/src/frontend/src/locales/nl/rooms.json b/src/frontend/src/locales/nl/rooms.jsonindex b6734794..aaa9f9d7 100644--- a/src/frontend/src/locales/nl/rooms.json+++ b/src/frontend/src/locales/nl/rooms.json@@ -237,6 +237,7 @@"description": "Met de beeld-in-beeld-modus kunt u verbonden blijven met het gesprek terwijl u andere taken uitvoert.","bringBack": "Gesprek hier terughalen"},+ "stage": "Deelnemers","controlBar": "Vergaderbesturing"},"options": {