Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

perf(app): defer media capture routing and skip offscreen skeleton layout (#1732)

Hampus698469fa96db
diff --git a/fluxer_app/src/features/app/components/skeleton/PlaceholderSpecs.ts b/fluxer_app/src/features/app/components/skeleton/PlaceholderSpecs.ts
index 0617e02ad..496f853b4 100644
--- a/fluxer_app/src/features/app/components/skeleton/PlaceholderSpecs.ts
+++ b/fluxer_app/src/features/app/components/skeleton/PlaceholderSpecs.ts
@@ -19,10 +19,12 @@ export interface PlaceholderMessageGroup {
readonly usernameWidth: number;
readonly timestampWidth: number;
readonly attachment: PlaceholderAttachmentSize | null;
+ readonly height: number;
}
interface MutablePlaceholderMessageGroup extends PlaceholderMessageGroup {
attachment: PlaceholderAttachmentSize | null;
+ height: number;
}
export interface PlaceholderSpecs {
@@ -100,20 +102,20 @@ function generatePlaceholderSpecs(options: PlaceholderGenerationOptions): Placeh
for (let line = 0; line < lineCount; line++) {
lineWidths.push(LINE_WIDTH_MIN + random() * LINE_WIDTH_RANGE);
}
+ const groupHeight = compact
+ ? MESSAGE_HEIGHT_COMPACT * lineCount
+ : COZY_LEAD_MESSAGE_HEIGHT + COZY_GROUPED_MESSAGE_HEIGHT * (lineCount - 1);
groups.push({
lineWidths,
usernameWidth: USERNAME_WIDTH_MIN + random() * USERNAME_WIDTH_RANGE,
timestampWidth: TIMESTAMP_WIDTH_MIN + random() * TIMESTAMP_WIDTH_RANGE,
attachment: null,
+ height: groupHeight,
});
if (index > 0) {
totalHeight += groupSpacing;
}
- if (compact) {
- totalHeight += MESSAGE_HEIGHT_COMPACT * lineCount;
- } else {
- totalHeight += COZY_LEAD_MESSAGE_HEIGHT + COZY_GROUPED_MESSAGE_HEIGHT * (lineCount - 1);
- }
+ totalHeight += groupHeight;
}
const availableGroupIndices = Array.from(Array(groups.length).keys());
for (let index = 0; index < attachments && availableGroupIndices.length > 0; index++) {
@@ -123,6 +125,7 @@ function generatePlaceholderSpecs(options: PlaceholderGenerationOptions): Placeh
height: randomInRange(random, ATTACHMENT_HEIGHT_MIN, ATTACHMENT_HEIGHT_MAX),
};
groups[groupIndex].attachment = attachment;
+ groups[groupIndex].height += attachment.height + ATTACHMENT_MARGIN;
totalHeight += attachment.height + ATTACHMENT_MARGIN;
}
return {compact, compactAvatarsVisible, groups, totalHeight, groupSpacing};
diff --git a/fluxer_app/src/features/app/components/skeleton/ScrollFillerSkeleton.module.css b/fluxer_app/src/features/app/components/skeleton/ScrollFillerSkeleton.module.css
index 2d4b228ac..f0d68947e 100644
--- a/fluxer_app/src/features/app/components/skeleton/ScrollFillerSkeleton.module.css
+++ b/fluxer_app/src/features/app/components/skeleton/ScrollFillerSkeleton.module.css
@@ -44,6 +44,7 @@
.messageGroup {
position: relative;
z-index: 0;
+ content-visibility: auto;
}
.group {
diff --git a/fluxer_app/src/features/app/components/skeleton/ScrollFillerSkeleton.tsx b/fluxer_app/src/features/app/components/skeleton/ScrollFillerSkeleton.tsx
index acca37819..0ef7ba97e 100644
--- a/fluxer_app/src/features/app/components/skeleton/ScrollFillerSkeleton.tsx
+++ b/fluxer_app/src/features/app/components/skeleton/ScrollFillerSkeleton.tsx
@@ -52,7 +52,10 @@ function CozyScrollFillerSkeletonGroup({
return (
<flx-message-list-skeleton-group
className={flxElementClassName(styles.messageGroup)}
- style={{marginBottom: resolveSkeletonGroupMarginBottom(groupIndex, groupCount, groupSpacing)}}
+ style={{
+ marginBottom: resolveSkeletonGroupMarginBottom(groupIndex, groupCount, groupSpacing),
+ containIntrinsicSize: `auto ${remFromPx(group.height)}`,
+ }}
data-flx="app.skeleton.scroll-filler-skeleton.cozy-scroll-filler-skeleton-group.message-group"
>
<flx-message-list-skeleton-group-row
@@ -170,7 +173,10 @@ function CompactScrollFillerSkeletonGroup({
return (
<flx-message-list-skeleton-group
className={flxElementClassName(styles.messageGroup)}
- style={{marginBottom: resolveSkeletonGroupMarginBottom(groupIndex, groupCount, groupSpacing)}}
+ style={{
+ marginBottom: resolveSkeletonGroupMarginBottom(groupIndex, groupCount, groupSpacing),
+ containIntrinsicSize: `auto ${remFromPx(group.height)}`,
+ }}
data-flx="app.skeleton.scroll-filler-skeleton.compact-scroll-filler-skeleton-group.message-group"
>
<flx-message-list-skeleton-compact-messages
diff --git a/fluxer_app/src/features/app/state/Initialization.ts b/fluxer_app/src/features/app/state/Initialization.ts
index 4f6dffd0e..33be943e7 100644
--- a/fluxer_app/src/features/app/state/Initialization.ts
+++ b/fluxer_app/src/features/app/state/Initialization.ts
@@ -16,7 +16,6 @@ class Initialization {
state: InitializationState = InitializationState.LOADING;
hasCompletedInitialLoad = false;
error: string | null = null;
- readyPayload: unknown = null;
constructor() {
makeAutoObservable(this, {}, {autoBind: true});
@@ -46,29 +45,25 @@ class Initialization {
setLoading(): void {
this.state = InitializationState.LOADING;
this.error = null;
- this.readyPayload = null;
}
@action
setConnecting(): void {
this.state = InitializationState.CONNECTING;
this.error = null;
- this.readyPayload = null;
}
@action
- setReady(payload: unknown): void {
+ setReady(): void {
this.state = InitializationState.READY;
this.hasCompletedInitialLoad = true;
this.error = null;
- this.readyPayload = payload;
}
@action
setError(error: string): void {
this.state = InitializationState.ERROR;
this.error = error;
- this.readyPayload = null;
}
@action
@@ -76,7 +71,6 @@ class Initialization {
this.state = InitializationState.LOADING;
this.hasCompletedInitialLoad = false;
this.error = null;
- this.readyPayload = null;
}
}
diff --git a/fluxer_app/src/features/gateway/events/GatewayReady.ts b/fluxer_app/src/features/gateway/events/GatewayReady.ts
index 504a2360f..f58f35865 100644
--- a/fluxer_app/src/features/gateway/events/GatewayReady.ts
+++ b/fluxer_app/src/features/gateway/events/GatewayReady.ts
@@ -183,7 +183,7 @@ function handleReadyInternal(data: ReadyPayload, context: GatewayHandlerContext)
GuildReadState.handleConnectionOpen();
Presence.handleConnectionOpen(data.user, guilds, data.presences);
MediaEngine.handleConnectionOpen(guilds);
- Initialization.setReady(data);
+ Initialization.setReady();
context.setReady();
Messages.handleConnectionOpen((channelId) => ReadStates.getUnreadJumpAnchor(channelId));
}
diff --git a/fluxer_app/src/features/notification/utils/SoundUtils.ts b/fluxer_app/src/features/notification/utils/SoundUtils.ts
index 53698dee8..e323fa477 100644
--- a/fluxer_app/src/features/notification/utils/SoundUtils.ts
+++ b/fluxer_app/src/features/notification/utils/SoundUtils.ts
@@ -236,10 +236,33 @@ export function getSoundCaptureMasterGainNode(): GainNode {
return getMasterGainNode();
}
+const soundCaptureActivationListeners: Set<() => void> = new Set();
+
+export function isSoundCaptureActive(): boolean {
+ return captureTaps.size > 0;
+}
+
+export function onSoundCaptureActivated(listener: () => void): () => void {
+ soundCaptureActivationListeners.add(listener);
+ return () => {
+ soundCaptureActivationListeners.delete(listener);
+ };
+}
+
export function addSoundCaptureDestination(node: AudioNode): void {
if (captureTaps.has(node)) return;
const master = getMasterGainNode();
+ const wasInactive = captureTaps.size === 0;
captureTaps.add(node);
+ if (wasInactive) {
+ for (const listener of soundCaptureActivationListeners) {
+ try {
+ listener();
+ } catch (error) {
+ logger.warn('Sound capture activation listener failed', error);
+ }
+ }
+ }
try {
master.connect(node);
} catch (error) {
diff --git a/fluxer_app/src/features/voice/utils/InAppMediaSoundCapture.test.ts b/fluxer_app/src/features/voice/utils/InAppMediaSoundCapture.test.ts
new file mode 100644
index 000000000..c649e8a82
--- /dev/null
+++ b/fluxer_app/src/features/voice/utils/InAppMediaSoundCapture.test.ts
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {beforeEach, describe, expect, it, vi} from 'vitest';
+
+const master = {connect: vi.fn(), disconnect: vi.fn()};
+const createdGains: Array<{connect: ReturnType<typeof vi.fn>; disconnect: ReturnType<typeof vi.fn>}> = [];
+let captureActive = true;
+
+vi.mock('@app/features/notification/utils/SoundUtils', () => ({
+ getSoundCaptureAudioContext: () => ({
+ createMediaElementSource: vi.fn(() => ({connect: vi.fn(), disconnect: vi.fn()})),
+ createGain: vi.fn(() => {
+ const gain = {connect: vi.fn(), disconnect: vi.fn(), gain: {value: 1}};
+ createdGains.push(gain);
+ return gain;
+ }),
+ }),
+ getSoundCaptureMasterGainNode: () => master,
+ isSoundCaptureActive: () => captureActive,
+ onSoundCaptureActivated: () => () => undefined,
+}));
+
+const {routeMediaElementForSoundCapture} = await import('@app/features/voice/utils/InAppMediaSoundCapture');
+
+function createElement(): HTMLMediaElement {
+ const listeners = new Map<string, () => void>();
+ return {
+ volume: 1,
+ muted: false,
+ addEventListener: (type: string, listener: () => void) => listeners.set(type, listener),
+ removeEventListener: (type: string) => listeners.delete(type),
+ listenerCount: () => listeners.size,
+ } as unknown as HTMLMediaElement & {listenerCount: () => number};
+}
+
+describe('in-app media sound capture', () => {
+ beforeEach(() => {
+ createdGains.length = 0;
+ captureActive = true;
+ });
+
+ it('disconnects the element from the master graph when the caller disposes', () => {
+ const element = createElement();
+ const dispose = routeMediaElementForSoundCapture(element);
+ const gain = createdGains.at(-1);
+ expect(gain?.connect).toHaveBeenCalledWith(master);
+ dispose();
+ expect(gain?.disconnect).toHaveBeenCalled();
+ });
+
+ it('removes the volume listener when the caller disposes', () => {
+ const element = createElement() as HTMLMediaElement & {listenerCount: () => number};
+ const dispose = routeMediaElementForSoundCapture(element);
+ expect(element.listenerCount()).toBe(1);
+ dispose();
+ expect(element.listenerCount()).toBe(0);
+ });
+
+ it('reuses the existing source node when the same element is routed again', () => {
+ const element = createElement();
+ routeMediaElementForSoundCapture(element)();
+ const gainsAfterFirst = createdGains.length;
+ const dispose = routeMediaElementForSoundCapture(element);
+ expect(createdGains.length).toBe(gainsAfterFirst);
+ expect(createdGains.at(-1)?.connect).toHaveBeenCalledTimes(2);
+ dispose();
+ });
+});
diff --git a/fluxer_app/src/features/voice/utils/InAppMediaSoundCapture.ts b/fluxer_app/src/features/voice/utils/InAppMediaSoundCapture.ts
index cd62ac3ae..25c2fc339 100644
--- a/fluxer_app/src/features/voice/utils/InAppMediaSoundCapture.ts
+++ b/fluxer_app/src/features/voice/utils/InAppMediaSoundCapture.ts
@@ -1,17 +1,24 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
-import {getSoundCaptureAudioContext, getSoundCaptureMasterGainNode} from '@app/features/notification/utils/SoundUtils';
+import {
+ getSoundCaptureAudioContext,
+ getSoundCaptureMasterGainNode,
+ isSoundCaptureActive,
+ onSoundCaptureActivated,
+} from '@app/features/notification/utils/SoundUtils';
import {Logger} from '@app/features/platform/utils/AppLogger';
const logger = new Logger('InAppMediaSoundCapture');
interface RoutedElement {
+ connected: boolean;
source: MediaElementAudioSourceNode;
gain: GainNode;
volumeListener: () => void;
}
const routedElements: WeakMap<HTMLMediaElement, RoutedElement> = new WeakMap();
+const pendingElements: Set<WeakRef<HTMLMediaElement>> = new Set();
function syncGainFromElement(el: HTMLMediaElement, gain: GainNode): void {
const volume = el.muted ? 0 : Math.max(0, Math.min(1, el.volume));
@@ -20,9 +27,20 @@ function syncGainFromElement(el: HTMLMediaElement, gain: GainNode): void {
} catch {}
}
-export function routeMediaElementForSoundCapture(element: HTMLMediaElement): () => void {
- if (routedElements.has(element)) {
- return () => undefined;
+function attachToCaptureGraph(element: HTMLMediaElement): void {
+ const existing = routedElements.get(element);
+ if (existing) {
+ if (existing.connected) return;
+ try {
+ existing.gain.connect(getSoundCaptureMasterGainNode());
+ } catch (error) {
+ logger.debug('Failed to reconnect media element to sound capture graph', {error});
+ return;
+ }
+ element.addEventListener('volumechange', existing.volumeListener);
+ syncGainFromElement(element, existing.gain);
+ existing.connected = true;
+ return;
}
let ctx: AudioContext;
let master: GainNode;
@@ -31,7 +49,7 @@ export function routeMediaElementForSoundCapture(element: HTMLMediaElement): ()
master = getSoundCaptureMasterGainNode();
} catch (error) {
logger.debug('Sound capture context unavailable; leaving media element on native playback', {error});
- return () => undefined;
+ return;
}
let source: MediaElementAudioSourceNode;
let gain: GainNode;
@@ -42,23 +60,45 @@ export function routeMediaElementForSoundCapture(element: HTMLMediaElement): ()
gain.connect(master);
} catch (error) {
logger.debug('Failed to route media element through sound capture graph', {error});
- return () => undefined;
+ return;
}
syncGainFromElement(element, gain);
const volumeListener = (): void => syncGainFromElement(element, gain);
element.addEventListener('volumechange', volumeListener);
- const routed: RoutedElement = {source, gain, volumeListener};
- routedElements.set(element, routed);
+ routedElements.set(element, {source, gain, volumeListener, connected: true});
+}
+
+function detachFromCaptureGraph(element: HTMLMediaElement): void {
+ const entry = routedElements.get(element);
+ if (!entry || !entry.connected) return;
+ entry.connected = false;
+ element.removeEventListener('volumechange', entry.volumeListener);
+ try {
+ entry.gain.disconnect();
+ } catch {}
+}
+
+function flushPendingElements(): void {
+ for (const ref of pendingElements) {
+ const element = ref.deref();
+ pendingElements.delete(ref);
+ if (element) {
+ attachToCaptureGraph(element);
+ }
+ }
+}
+
+onSoundCaptureActivated(flushPendingElements);
+
+export function routeMediaElementForSoundCapture(element: HTMLMediaElement): () => void {
+ if (isSoundCaptureActive()) {
+ attachToCaptureGraph(element);
+ return () => detachFromCaptureGraph(element);
+ }
+ const ref = new WeakRef(element);
+ pendingElements.add(ref);
return () => {
- const entry = routedElements.get(element);
- if (!entry || entry !== routed) return;
- routedElements.delete(element);
- element.removeEventListener('volumechange', volumeListener);
- try {
- gain.disconnect();
- } catch {}
- try {
- source.disconnect();
- } catch {}
+ pendingElements.delete(ref);
+ detachFromCaptureGraph(element);
};
}
diff --git a/fluxer_app/src/features/voice/utils/MediaDeviceStartupPreload.ts b/fluxer_app/src/features/voice/utils/MediaDeviceStartupPreload.ts
index d3f765bb1..3c8015c37 100644
--- a/fluxer_app/src/features/voice/utils/MediaDeviceStartupPreload.ts
+++ b/fluxer_app/src/features/voice/utils/MediaDeviceStartupPreload.ts
@@ -27,7 +27,6 @@ export function startMediaDeviceStartupPreload(): () => void {
logger.debug('Failed to preload media devices', {error});
});
};
- preloadDevices();
const disposePermissionListener = MediaPermission.addChangeListener(preloadDevices);
return () => {
stopped = true;