Skip to content

armgit / meet

Public Imported from GitHub
0

Open source video conferencing app powered by LiveKit. Built with Django and React.

✨(frontend) introduce the camera stage

lebaudantoine38c131e02cff
diff --git a/src/frontend/src/features/pip/components/PipView.tsx b/src/frontend/src/features/pip/components/PipView.tsx
index da463acc..c99cb95f 100644
--- a/src/frontend/src/features/pip/components/PipView.tsx
+++ b/src/frontend/src/features/pip/components/PipView.tsx
@@ -1,7 +1,9 @@
+import { styled } from '@/styled-system/jsx'
import { PipControlBar } from './PipControlBar'
import { PipFloatingReactions } from './PipFloatingReactions'
+import { PipStage } from './layout/PipStage'
import { ReactionsToolbar } from '@/features/reactions/components/toolbar/ReactionsToolbar'
-import { styled } from '@/styled-system/jsx'
+import { useReactionsToolbar } from '@/features/reactions/hooks/useReactionsToolbar'
const Container = styled('div', {
base: {
@@ -11,6 +13,7 @@ const Container = styled('div', {
display: 'grid',
gridTemplateRows: 'minmax(0, 1fr) auto auto',
backgroundColor: 'primaryDark.50',
+ transition: 'padding .5s cubic-bezier(0.4,0,0.2,1) 5ms',
// Disable LiveKit's own border-radius on tiles so our containers
// (GridCell, Thumbnail, StageFrame) own the clipping exclusively.
'--lk-border-radius': '4px',
@@ -29,11 +32,24 @@ const Container = styled('div', {
width: '100%',
},
},
+ variants: {
+ isReactionToolbarOpen: {
+ true: {
+ paddingBottom:
+ 'calc(var(--sizes-room-reaction-toolbar-height) + var(--sizes-room-control-bar) + 1.125rem)',
+ },
+ false: {
+ paddingBottom: 'var(--sizes-room-control-bar)',
+ },
+ },
+ },
})
export const PipView = () => {
+ const { isOpen: isReactionToolbarOpen } = useReactionsToolbar()
return (
- <Container>
+ <Container isReactionToolbarOpen={isReactionToolbarOpen}>
+ <PipStage />
<ReactionsToolbar adjustedCentering={false} />
<PipControlBar showScreenShare={false} />
<PipFloatingReactions />
diff --git a/src/frontend/src/features/pip/components/layout/PipFocusLayout.tsx b/src/frontend/src/features/pip/components/layout/PipFocusLayout.tsx
new file mode 100644
index 00000000..e09b5ef0
--- /dev/null
+++ b/src/frontend/src/features/pip/components/layout/PipFocusLayout.tsx
@@ -0,0 +1,81 @@
+import { memo } from 'react'
+import type { TrackReferenceOrPlaceholder } from '@livekit/components-core'
+import { styled } from '@/styled-system/jsx'
+import { ParticipantTile } from '@/features/rooms/livekit/components/ParticipantTile'
+import { getTrackKey } from '../../utils/pipTrackSelection'
+
+type PipFocusLayoutProps = {
+ mainTrack?: TrackReferenceOrPlaceholder
+ thumbnailTrack?: TrackReferenceOrPlaceholder
+}
+
+/**
+ * Focus layout used when 1-2 tracks are visible in the PiP window.
+ *
+ * The main tile is letterboxed (object-fit: contain) so the camera is
+ * never stretched to a non-video aspect and leaves dark padding
+ * above/below when the window shape doesn't match the source.
+ * The thumbnail keeps the usual cover fill.
+ */
+export const PipFocusLayout = memo(
+ ({ mainTrack, thumbnailTrack }: PipFocusLayoutProps) => {
+ return (
+ <FocusContainer>
+ {mainTrack && (
+ <MainSlot>
+ <ParticipantTile
+ key={getTrackKey(mainTrack)}
+ trackRef={mainTrack}
+ />
+ </MainSlot>
+ )}
+ {thumbnailTrack && (
+ <Thumbnail>
+ <ParticipantTile
+ key={getTrackKey(thumbnailTrack)}
+ trackRef={thumbnailTrack}
+ />
+ </Thumbnail>
+ )}
+ </FocusContainer>
+ )
+ }
+)
+PipFocusLayout.displayName = 'PipFocusLayout'
+
+const FocusContainer = styled('div', {
+ base: {
+ position: 'relative',
+ width: '100%',
+ height: '100%',
+ borderRadius: '4px',
+ overflow: 'hidden',
+ backgroundColor: 'primaryDark.100',
+ },
+})
+
+const MainSlot = styled('div', {
+ base: {
+ width: '100%',
+ height: '100%',
+ '& .lk-participant-media-video': {
+ objectFit: 'contain',
+ },
+ },
+})
+
+const Thumbnail = styled('div', {
+ base: {
+ position: 'absolute',
+ right: '1rem',
+ bottom: '1rem',
+ width: '42%',
+ maxWidth: '220px',
+ minWidth: '140px',
+ aspectRatio: '16 / 9',
+ borderRadius: '4px',
+ overflow: 'hidden',
+ boxShadow: 'md',
+ zIndex: 2,
+ },
+})
diff --git a/src/frontend/src/features/pip/components/layout/PipGridLayout.tsx b/src/frontend/src/features/pip/components/layout/PipGridLayout.tsx
new file mode 100644
index 00000000..5221ef85
--- /dev/null
+++ b/src/frontend/src/features/pip/components/layout/PipGridLayout.tsx
@@ -0,0 +1,76 @@
+import { memo, useMemo, useRef } from 'react'
+import type { TrackReferenceOrPlaceholder } from '@livekit/components-core'
+import { styled } from '@/styled-system/jsx'
+import { ParticipantTile } from '@/features/rooms/livekit/components/ParticipantTile'
+import { usePipElementSize } from '../../hooks/usePipElementSize'
+import { computePipGridLayout } from '../../utils/pipGrid'
+import { getTrackKey } from '../../utils/pipTrackSelection'
+
+type PipGridLayoutProps = {
+ tracks: TrackReferenceOrPlaceholder[]
+}
+
+/**
+ * Adaptive grid used when 3+ tracks are visible in the PiP window.
+ *
+ * All grid math (shape choice + partial-row stretching) is delegated to
+ * `computePipGridLayout`. This component only measures the container,
+ * applies the returned placements, and plays a FLIP animation when the
+ * tile set or grid shape changes (participant joins/leaves or shape shift).
+ *
+ * Tiles keep a stable key so resizing never remounts <video> elements.
+ */
+export const PipGridLayout = memo(({ tracks }: PipGridLayoutProps) => {
+ const containerRef = useRef<HTMLDivElement>(null)
+ const { width, height } = usePipElementSize(containerRef)
+
+ const { rows, subColumns, placements } = useMemo(
+ () => computePipGridLayout(tracks.length, width, height),
+ [tracks.length, width, height]
+ )
+
+ const gridStyle = useMemo(
+ () => ({
+ gridTemplateColumns: `repeat(${subColumns}, minmax(0, 1fr))`,
+ gridTemplateRows: `repeat(${rows}, minmax(0, 1fr))`,
+ }),
+ [subColumns, rows]
+ )
+
+ return (
+ <GridContainer ref={containerRef} style={gridStyle}>
+ {tracks.map((track, index) => (
+ <GridCell key={getTrackKey(track)} style={placements[index]}>
+ <ParticipantTile trackRef={track} />
+ </GridCell>
+ ))}
+ </GridContainer>
+ )
+})
+PipGridLayout.displayName = 'PipGridLayout'
+
+const GridContainer = styled('div', {
+ base: {
+ width: '100%',
+ height: '100%',
+ display: 'grid',
+ gap: '0.25rem',
+ },
+})
+
+const GridCell = styled('div', {
+ base: {
+ position: 'relative',
+ minWidth: 0,
+ minHeight: 0,
+ borderRadius: '4px',
+ overflow: 'hidden',
+ backgroundColor: 'primaryDark.100',
+ // Paint on own layer so FLIP transforms don't trigger layout thrash.
+ willChange: 'transform',
+ '& .lk-participant-tile': {
+ width: '100%',
+ height: '100%',
+ },
+ },
+})
diff --git a/src/frontend/src/features/pip/components/layout/PipStage.tsx b/src/frontend/src/features/pip/components/layout/PipStage.tsx
new file mode 100644
index 00000000..b59ffe7e
--- /dev/null
+++ b/src/frontend/src/features/pip/components/layout/PipStage.tsx
@@ -0,0 +1,77 @@
+import { useMemo } from 'react'
+import { useTracks } from '@livekit/components-react'
+import { RoomEvent, Track } from 'livekit-client'
+import { PipFocusLayout } from './PipFocusLayout'
+import { PipGridLayout } from './PipGridLayout'
+import { StageFrame } from './StageFrame'
+import {
+ isTrackReference,
+ TrackReferenceOrPlaceholder,
+} from '@livekit/components-core'
+
+/**
+ * PipStage picks between two layouts based on track count:
+ * - Focus mode (≤ 2 tracks): one main track + one thumbnail overlay.
+ * - Grid mode (3+ tracks): adaptive tiling.
+ */
+export const PipStage = () => {
+ const tracks = useTracks(
+ [
+ { source: Track.Source.Camera, withPlaceholder: true },
+ { source: Track.Source.ScreenShare, withPlaceholder: false },
+ ],
+ { updateOnlyOn: [RoomEvent.ActiveSpeakersChanged], onlySubscribed: false }
+ )
+
+ const screenShareTrack = useMemo(() => {
+ return tracks
+ .filter((track) => isTrackReference(track))
+ .find((track) => track.publication.source === Track.Source.ScreenShare)
+ }, [tracks])
+
+ const cameraTracks = useMemo(
+ () =>
+ tracks.filter(
+ (track: TrackReferenceOrPlaceholder) =>
+ track.source === Track.Source.Camera
+ ),
+ [tracks]
+ )
+
+ if (tracks.length === 0) return null
+
+ /**
+ * The focus layout shows one main track with one thumbnail overlay,
+ * so it can only fit 2 tracks. Beyond that we switch to the grid.
+ */
+ if (tracks.length > 2) {
+ // Grid mode: 3+ tracks. Screen share goes first so it leads the grid.
+ const gridTracks = screenShareTrack
+ ? [screenShareTrack, ...cameraTracks]
+ : cameraTracks
+ return (
+ <StageFrame>
+ <PipGridLayout tracks={gridTracks} />
+ </StageFrame>
+ )
+ }
+
+ const localCameraTrack = cameraTracks.find(
+ (track) => track.participant?.isLocal
+ )
+
+ const remoteCameraTrack = cameraTracks.find(
+ (track) => !track.participant?.isLocal
+ )
+
+ const mainTrack = screenShareTrack ?? remoteCameraTrack ?? localCameraTrack
+
+ const thumbnailTrack =
+ mainTrack === localCameraTrack ? undefined : localCameraTrack
+
+ return (
+ <StageFrame>
+ <PipFocusLayout mainTrack={mainTrack} thumbnailTrack={thumbnailTrack} />
+ </StageFrame>
+ )
+}
diff --git a/src/frontend/src/features/pip/components/layout/StageFrame.tsx b/src/frontend/src/features/pip/components/layout/StageFrame.tsx
new file mode 100644
index 00000000..46e34347
--- /dev/null
+++ b/src/frontend/src/features/pip/components/layout/StageFrame.tsx
@@ -0,0 +1,25 @@
+import { useTranslation } from 'react-i18next'
+import { styled } from '@/styled-system/jsx'
+
+export const StageFrame = ({ children }: { children: React.ReactNode }) => {
+ const { t } = useTranslation('rooms', {
+ keyPrefix: 'pictureInPicture',
+ })
+ return (
+ <Container role="region" aria-label={t('stage')} {...{ inert: '' }}>
+ {children}
+ </Container>
+ )
+}
+
+const Container = styled('div', {
+ base: {
+ position: 'relative',
+ minWidth: 0,
+ minHeight: 0,
+ marginLeft: '0.5rem',
+ marginRight: '0.5rem',
+ borderRadius: '4px',
+ overflow: 'hidden',
+ },
+})
diff --git a/src/frontend/src/features/pip/hooks/usePictureInPicture.ts b/src/frontend/src/features/pip/hooks/usePictureInPicture.ts
index 6842d0de..313d21d0 100644
--- a/src/frontend/src/features/pip/hooks/usePictureInPicture.ts
+++ b/src/frontend/src/features/pip/hooks/usePictureInPicture.ts
@@ -1,8 +1,10 @@
import { ref, useSnapshot } from 'valtio'
import { useCallback, useMemo } from 'react'
-import { IS_PIP_SUPPORTED } from '@/features/pip/utils'
import { documentPictureInPictureStore } from '@/stores/documentPictureInPicture'
+export const IS_PIP_SUPPORTED =
+ typeof globalThis !== 'undefined' && 'documentPictureInPicture' in globalThis
+
export const usePictureInPicture = () => {
const { window: pipWindowRef } = useSnapshot(documentPictureInPictureStore)
diff --git a/src/frontend/src/features/pip/hooks/usePipElementSize.ts b/src/frontend/src/features/pip/hooks/usePipElementSize.ts
new file mode 100644
index 00000000..f37042de
--- /dev/null
+++ b/src/frontend/src/features/pip/hooks/usePipElementSize.ts
@@ -0,0 +1,42 @@
+import { useCallback, useEffect, useState, type RefObject } from 'react'
+
+type Size = { width: number; height: number }
+
+/**
+ * Observes an element's size, even when mounted in the PiP document.
+ * Resolves `ResizeObserver` from the element's own window.
+ */
+export const usePipElementSize = <T extends HTMLElement>(
+ ref: RefObject<T | null>
+): Size => {
+ const [size, setSize] = useState<Size>({ width: 0, height: 0 })
+
+ const measure = useCallback(() => {
+ const el = ref.current
+ if (!el) return
+ const rect = el.getBoundingClientRect()
+ setSize({ width: rect.width, height: rect.height })
+ }, [ref])
+
+ useEffect(() => {
+ const el = ref.current
+ if (!el) return
+
+ measure()
+
+ const RO =
+ el.ownerDocument.defaultView?.ResizeObserver ?? globalThis.ResizeObserver
+ if (!RO) return
+
+ const observer = new RO((entries) => {
+ const entry = entries[0]
+ if (!entry) return
+ const { width, height } = entry.contentRect
+ setSize({ width, height })
+ })
+ observer.observe(el)
+ return () => observer.disconnect()
+ }, [ref, measure])
+
+ return size
+}
diff --git a/src/frontend/src/features/pip/utils.ts b/src/frontend/src/features/pip/utils.ts
deleted file mode 100644
index b333fedb..00000000
--- a/src/frontend/src/features/pip/utils.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-export const IS_PIP_SUPPORTED =
- typeof globalThis !== 'undefined' && 'documentPictureInPicture' in globalThis
diff --git a/src/frontend/src/features/pip/utils/pipGrid.ts b/src/frontend/src/features/pip/utils/pipGrid.ts
new file mode 100644
index 00000000..ce73dcdc
--- /dev/null
+++ b/src/frontend/src/features/pip/utils/pipGrid.ts
@@ -0,0 +1,114 @@
+export type PipTilePlacement = {
+ gridColumn: string
+ gridRow: number
+}
+
+export type PipGridLayout = {
+ cols: number
+ rows: number
+ /** Number of CSS sub-columns; use as `repeat(subColumns, 1fr)`. */
+ subColumns: number
+ /** One entry per tile, in input order. */
+ placements: PipTilePlacement[]
+}
+
+/**
+ * Target tile aspect ratio used to score candidate grid shapes.
+ *
+ * Video sources are 16:9, but picking 16:9 as the target makes the
+ * scorer indifferent between a stretched 2-col slab (aspect ~2.7) and a
+ * squarer 3-col tile (aspect ~1.2) because log distance is symmetric.
+ * The UI works better with square, face-friendly tiles. This target keeps
+ * wide windows from collapsing to 2 columns with short, stretched rows
+ * and pushes the scorer to add a column instead.
+ */
+const TARGET_TILE_ASPECT = 1
+
+/**
+ * Smallest count from which we force at least two columns.
+ * For 1-3 participants it is acceptable to stack vertically in tall
+ * windows, but from 4 people onwards we keep >=2 columns to
+ * avoid endless vertical scrolling; the scorer handles the rest.
+ */
+const FORCE_TWO_COLS_COUNT = 4
+
+const pickGridShape = (
+ count: number,
+ width: number,
+ height: number
+): { cols: number; rows: number } => {
+ if (count <= 1) return { cols: 1, rows: Math.max(1, count) }
+ if (width <= 0 || height <= 0) return { cols: count, rows: 1 }
+
+ const minCols = count >= FORCE_TWO_COLS_COUNT ? 2 : 1
+
+ let best = {
+ cols: minCols,
+ rows: Math.ceil(count / minCols),
+ score: -Infinity,
+ }
+ for (let cols = minCols; cols <= count; cols++) {
+ const rows = Math.ceil(count / cols)
+ const tileW = width / cols
+ const tileH = height / rows
+ if (tileW <= 0 || tileH <= 0) continue
+
+ // Score: aspect close to target, few empty cells, large tile area,
+ // and a tiny bias toward fewer rows so ties (perfectly square shapes)
+ // resolve in favour of a shorter, wider grid.
+ const aspectScore = -Math.abs(Math.log(tileW / tileH / TARGET_TILE_ASPECT))
+ const emptyCells = cols * rows - count
+ const fillScore = -emptyCells * 0.1
+ const areaScore = Math.log(tileW * tileH) * 0.5
+ const rowsPenalty = -rows * 0.01
+
+ const score = aspectScore * 2 + fillScore + areaScore + rowsPenalty
+ if (score > best.score) best = { cols, rows, score }
+ }
+ return { cols: best.cols, rows: best.rows }
+}
+
+/**
+ * Pure function. Given a tile count and stage dimensions, returns the CSS
+ * grid layout for the PiP stage:
+ *
+ * - picks a cols x rows shape close to 16:9 tiles,
+ * - stretches any partial last row so its tiles share the full row width
+ * (no empty cells, no small centered tile).
+ *
+ * Callers consume the result directly: `subColumns` feeds
+ * `grid-template-columns: repeat(N, 1fr)` and each tile reads its own
+ * `gridColumn`/`gridRow` from `placements`.
+ */
+export const computePipGridLayout = (
+ count: number,
+ width: number,
+ height: number
+): PipGridLayout => {
+ if (count <= 0) {
+ return { cols: 1, rows: 1, subColumns: 1, placements: [] }
+ }
+
+ const { cols, rows } = pickGridShape(count, width, height)
+ const tilesInLastRow = count - cols * (rows - 1)
+ const hasPartialRow = tilesInLastRow > 0 && tilesInLastRow < cols
+
+ const subColumns = hasPartialRow ? cols * tilesInLastRow : cols
+ const fullRowSpan = hasPartialRow ? tilesInLastRow : 1
+ const lastRowSpan = hasPartialRow ? cols : 1
+
+ const placements: PipTilePlacement[] = []
+ for (let i = 0; i < count; i++) {
+ const row = Math.floor(i / cols)
+ const colIndex = i % cols
+ const isLastRow = row === rows - 1 && hasPartialRow
+ const span = isLastRow ? lastRowSpan : fullRowSpan
+ const colStart = colIndex * span + 1
+ placements.push({
+ gridColumn: `${colStart} / span ${span}`,
+ gridRow: row + 1,
+ })
+ }
+
+ return { cols, rows, subColumns, placements }
+}
diff --git a/src/frontend/src/features/pip/utils/pipTrackSelection.ts b/src/frontend/src/features/pip/utils/pipTrackSelection.ts
new file mode 100644
index 00000000..96062922
--- /dev/null
+++ b/src/frontend/src/features/pip/utils/pipTrackSelection.ts
@@ -0,0 +1,16 @@
+import {
+ isTrackReference,
+ TrackReferenceOrPlaceholder,
+} from '@livekit/components-core'
+
+/**
+ * Produces a stable React key for a track so resizes/reshuffles of the grid
+ * do not remount the underlying <video> element.
+ */
+export const getTrackKey = (track: TrackReferenceOrPlaceholder): string => {
+ const identity = track.participant?.identity ?? 'unknown'
+ if (isTrackReference(track)) {
+ return `${identity}::${track.source}::${track.publication.trackSid}`
+ }
+ return `${identity}::${track.source}::placeholder`
+}
diff --git a/src/frontend/src/locales/de/rooms.json b/src/frontend/src/locales/de/rooms.json
index 07eadd20..ea8b6454 100644
--- a/src/frontend/src/locales/de/rooms.json
+++ b/src/frontend/src/locales/de/rooms.json
@@ -237,6 +237,7 @@
"description": "Im Bild-im-Bild-Modus bleiben Sie mit dem Anruf verbunden, während Sie andere Aufgaben erledigen.",
"bringBack": "Anruf hierher zurückholen"
},
+ "stage": "Teilnehmer",
"controlBar": "Besprechungssteuerung"
},
"options": {
diff --git a/src/frontend/src/locales/en/rooms.json b/src/frontend/src/locales/en/rooms.json
index 1c724203..17afa83e 100644
--- a/src/frontend/src/locales/en/rooms.json
+++ b/src/frontend/src/locales/en/rooms.json
@@ -237,6 +237,7 @@
"description": "Picture-in-Picture mode allows you to stay connected to the call while performing other tasks.",
"bringBack": "Bring the call back here"
},
+ "stage": "Participants",
"controlBar": "Meeting controls"
},
"options": {
diff --git a/src/frontend/src/locales/fr/rooms.json b/src/frontend/src/locales/fr/rooms.json
index 0f03f877..f408c50f 100644
--- a/src/frontend/src/locales/fr/rooms.json
+++ b/src/frontend/src/locales/fr/rooms.json
@@ -237,6 +237,7 @@
"description": "Le mode image dans l'image vous permet de rester connecté à l'appel tout en effectuant d'autres tâches.",
"bringBack": "Ramener l'appel ici"
},
+ "stage": "Participants",
"controlBar": "Commandes de la réunion"
},
"options": {
diff --git a/src/frontend/src/locales/nl/rooms.json b/src/frontend/src/locales/nl/rooms.json
index b6734794..aaa9f9d7 100644
--- a/src/frontend/src/locales/nl/rooms.json
+++ b/src/frontend/src/locales/nl/rooms.json
@@ -237,6 +237,7 @@
"description": "Met de beeld-in-beeld-modus kunt u verbonden blijven met het gesprek terwijl u andere taken uitvoert.",
"bringBack": "Gesprek hier terughalen"
},
+ "stage": "Deelnemers",
"controlBar": "Vergaderbesturing"
},
"options": {