Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(remote-desktop): 音频采集失败时保留视频并自动重试 (#4613)

Dash3cbdd4fddd2f
diff --git a/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts b/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts
index 03ec766a2a..6a43b6aa46 100644
--- a/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts
+++ b/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts
@@ -1,5 +1,9 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
-import { DESKTOP_LOCAL } from '../../../shared/remoteDesktop';
+import { DESKTOP_AUDIO_RETRY_MS, DESKTOP_LOCAL } from '../../../shared/remoteDesktop';
+
+vi.hoisted(() => {
+ vi.stubGlobal('process', { ...process, platform: 'darwin', getSystemVersion: () => '26.0' });
+});
const h = vi.hoisted(() => ({
handlers: new Map<string, any>(),
@@ -631,3 +635,87 @@ it('does not advertise or select Windows overlays without a ready native service
h.handlers.get(DESKTOP_LOCAL.REPLY)(event(), command.id, 'answer');
await pending;
});
+
+it.each([true, false])(
+ 'bounds same-screen audio recovery after the initial grant was consumed=%s',
+ async (consumed) => {
+ const pending = h.deps.offer(
+ { lease: h.lease, display: { id: '1' } },
+ 'sdp',
+ { audio: true, fps: 30, bitrate: 0 },
+ true,
+ 'attempt',
+ );
+ h.handlers.get(DESKTOP_LOCAL.REGISTER)(event());
+ await flush();
+ const owner = h.owner;
+ const grant = owner.session.setDisplayMediaRequestHandler.mock.calls[0][0];
+ const callback = vi.fn();
+ const request = { frame: owner.mainFrame, videoRequested: true, audioRequested: true };
+ if (consumed) {
+ grant(request, callback);
+ expect(callback).toHaveBeenLastCalledWith({
+ video: { id: 'screen:1', display_id: '1' },
+ audio: 'loopback',
+ });
+ }
+ h.handlers.get(DESKTOP_LOCAL.REPLY)(event(), owner.send.mock.calls[0][1].id, 'answer');
+ await pending;
+ // An OS denial can happen before the first display grant is consumed.
+ // The answered video lease still permits only bounded audio recovery.
+ for (const invalid of [
+ { ...request, frame: {} },
+ { ...request, audioRequested: false },
+ { ...request, videoRequested: false },
+ ]) {
+ grant(invalid, callback);
+ expect(callback).toHaveBeenLastCalledWith({});
+ }
+ for (let i = consumed ? 1 : 0; i < 1 + DESKTOP_AUDIO_RETRY_MS.length; i++) {
+ grant(request, callback);
+ expect(callback).toHaveBeenLastCalledWith({
+ video: { id: 'screen:1', display_id: '1' },
+ audio: 'loopback',
+ });
+ }
+ grant(request, callback);
+ expect(callback).toHaveBeenLastCalledWith({});
+ expect(owner.dead).toBe(false);
+ },
+);
+
+it('revokes audio recovery with the lease and never grants it to an audio-off replacement', async () => {
+ const pending = h.deps.offer(
+ { lease: h.lease, display: { id: '1' } },
+ 'sdp',
+ { audio: true, fps: 30, bitrate: 0 },
+ true,
+ 'attempt',
+ );
+ h.handlers.get(DESKTOP_LOCAL.REGISTER)(event());
+ await flush();
+ const oldOwner = h.owner;
+ const oldGrant = oldOwner.session.setDisplayMediaRequestHandler.mock.calls[0][0];
+ h.handlers.get(DESKTOP_LOCAL.REPLY)(event(), oldOwner.send.mock.calls[0][1].id, 'answer');
+ await pending;
+ h.lease = 'replacement';
+ const callback = vi.fn();
+ const oldRequest = { frame: oldOwner.mainFrame, videoRequested: true, audioRequested: true };
+ oldGrant(oldRequest, callback);
+ expect(callback).toHaveBeenLastCalledWith({});
+ const replacement = offer();
+ h.handlers.get(DESKTOP_LOCAL.REGISTER)(event());
+ await flush();
+ const owner = h.owner;
+ const grant = owner.session.setDisplayMediaRequestHandler.mock.calls[0][0];
+ oldGrant(oldRequest, callback);
+ expect(callback).toHaveBeenLastCalledWith({});
+ const request = { ...oldRequest, frame: owner.mainFrame };
+ grant(request, callback);
+ expect(callback).toHaveBeenLastCalledWith({ video: { id: 'screen:1', display_id: '1' } });
+ h.handlers.get(DESKTOP_LOCAL.REPLY)(event(), owner.send.mock.calls[0][1].id, 'answer');
+ await replacement;
+ grant(request, callback);
+ expect(callback).toHaveBeenLastCalledWith({});
+ expect(owner.dead).toBe(false);
+});
diff --git a/apps/desktop/src/main/remote-desktop/index.ts b/apps/desktop/src/main/remote-desktop/index.ts
index 45aca3ba99..3271d0f740 100644
--- a/apps/desktop/src/main/remote-desktop/index.ts
+++ b/apps/desktop/src/main/remote-desktop/index.ts
@@ -26,6 +26,7 @@ import {
} from '@cindy/device-link';
import {
DESKTOP_LOCAL,
+ DESKTOP_AUDIO_RETRY_MS,
type DesktopHostCommand,
type DesktopHostReply,
} from '../../shared/remoteDesktop';
@@ -120,7 +121,13 @@ const supportsPrivacyScreen =
let displayAwake: number | null = null;
let nativeDisplay: string | null = null;
let windowsAvailable = false;
-let captureGrant: { source: DesktopCapturerSource; lease: string; audio: boolean } | null = null;
+let captureGrant: {
+ source: DesktopCapturerSource;
+ lease: string;
+ audio: boolean;
+ remaining: number;
+ used: boolean;
+} | null = null;
const supportsSystemAudio =
process.platform === 'win32' ||
(process.platform === 'darwin' &&
@@ -232,7 +239,15 @@ async function offer(
)
throw new Error('DESKTOP_LEASE_EXPIRED');
if (settings?.audio && !supportsSystemAudio) throw new Error('DESKTOP_AUDIO_UNAVAILABLE');
- captureGrant = source ? { source, lease: lease.lease, audio: settings?.audio === true } : null;
+ captureGrant = source
+ ? {
+ source,
+ lease: lease.lease,
+ audio: settings?.audio === true,
+ remaining: settings?.audio ? 1 + DESKTOP_AUDIO_RETRY_MS.length : 1,
+ used: false,
+ }
+ : null;
nativeDisplay = nativeAvailable ? lease.display.id : null;
nativeOverlay = cursorOverlay === true && nativeAvailable;
nativeSettings = settings;
@@ -364,7 +379,8 @@ export const remoteDesktop = new RemoteDesktopController({
const viewerDisplay = enabled && (await viewerDisplaySupported());
return {
version: 1,
- cursorOverlay: process.platform === 'darwin' || (process.platform === 'win32' && windowsAvailable),
+ cursorOverlay:
+ process.platform === 'darwin' || (process.platform === 'win32' && windowsAvailable),
lockOnExit: process.platform === 'darwin',
clipboardContent: process.platform === 'darwin' || process.platform === 'win32',
clipboardSync: process.platform === 'darwin' || process.platform === 'win32',
@@ -405,11 +421,7 @@ export const remoteDesktop = new RemoteDesktopController({
// Compatibility viewers must also wake/capture without waiting for
// Chromium's thumbnail enumeration, which may hang on a sleeping display.
if (process.platform === 'darwin' || windowsAvailable) {
- const frame = await nativeCapture.frame(
- displayId,
- cursorOverlay === true,
- nativeSettings,
- );
+ const frame = await nativeCapture.frame(displayId, cursorOverlay === true, nativeSettings);
return encodeNativeRelayFrame(frame, (jpeg) => nativeImage.createFromBuffer(jpeg));
}
const available = await sources(true).catch((error) => {
@@ -674,14 +686,17 @@ export function registerRemoteDesktopIpc(
host !== owner ||
request.frame !== owner.mainFrame ||
!remoteDesktop.hasLease(grant.lease) ||
- !pending ||
- pending.op !== 'offer' ||
+ videoLease !== grant.lease ||
+ ((grant.used || pending?.op !== 'offer') && (!grant.audio || !request.audioRequested)) ||
!request.videoRequested
) {
callback({});
return;
}
- captureGrant = null;
+ // Reuse only this screen/lease's audio-enabled grant, never a general
+ // display picker grant. OS permission is checked on every capture.
+ grant.used = true;
+ if (--grant.remaining === 0) captureGrant = null;
callback({
video: grant.source,
...(grant.audio && request.audioRequested ? { audio: 'loopback' as const } : {}),
diff --git a/apps/desktop/src/renderer/features/remote-desktop/__tests__/RemoteDesktopHost.test.tsx b/apps/desktop/src/renderer/features/remote-desktop/__tests__/RemoteDesktopHost.test.tsx
index 0528fcff5a..de5e34ddb4 100644
--- a/apps/desktop/src/renderer/features/remote-desktop/__tests__/RemoteDesktopHost.test.tsx
+++ b/apps/desktop/src/renderer/features/remote-desktop/__tests__/RemoteDesktopHost.test.tsx
@@ -112,61 +112,6 @@ it('keeps system audio with cursor-free native video and stops cursor updates wi
expect(stopNative).toHaveBeenCalledOnce();
});
-it.each(['rejected', 'missing'] as const)(
- 'rejects native video with %s requested audio and releases capture',
- async (audio) => {
- const track = { stop: vi.fn() };
- const stream = {
- getTracks: () => [track],
- getVideoTracks: () => [track],
- getAudioTracks: () => [],
- addTrack: vi.fn(),
- };
- const stopNative = vi.fn();
- vi.mocked(nativeCaptureStream).mockResolvedValue({
- stream,
- stop: stopNative,
- } as unknown as Awaited<ReturnType<typeof nativeCaptureStream>>);
- const capture =
- audio === 'rejected'
- ? vi.fn().mockRejectedValue(new Error('unavailable'))
- : vi.fn().mockResolvedValue(stream);
- vi.stubGlobal('navigator', { mediaDevices: { getDisplayMedia: capture } });
- let command!: (value: unknown) => void;
- const reply = vi.fn().mockResolvedValue(undefined);
- Object.assign(window, {
- electronAPI: {
- remoteDesktop: {
- onCommand: (callback: typeof command) => {
- command = callback;
- return () => {};
- },
- registerHost: vi.fn().mockResolvedValue(undefined),
- state: vi.fn().mockResolvedValue(null),
- stop: vi.fn().mockResolvedValue(undefined),
- reply,
- },
- },
- });
- disposers.push(startDesktopCaptureHost(window.electronAPI.remoteDesktop as any));
- await act(async () => {
- command({
- op: 'offer',
- id: 'offer',
- lease: 'lease',
- sdp: 'sdp',
- sourceId: 'screen:1',
- nativeCapture: true,
- cursorOverlay: true,
- settings: { audio: true, fps: 30 },
- });
- });
- expect(reply).toHaveBeenCalledWith('offer', { error: 'DESKTOP_AUDIO_UNAVAILABLE' });
- expect(stopNative).toHaveBeenCalled();
- expect(track.stop).toHaveBeenCalled();
- },
-);
-
it('exchanges replayable candidates without recapturing, tolerates transient disconnect and fences old attempts', async () => {
vi.useFakeTimers();
const peers: any[] = [];
diff --git a/apps/desktop/src/renderer/features/remote-desktop/__tests__/captureAudioRecovery.test.ts b/apps/desktop/src/renderer/features/remote-desktop/__tests__/captureAudioRecovery.test.ts
new file mode 100644
index 0000000000..8aeda9d618
--- /dev/null
+++ b/apps/desktop/src/renderer/features/remote-desktop/__tests__/captureAudioRecovery.test.ts
@@ -0,0 +1,227 @@
+// @vitest-environment jsdom
+import { afterEach, expect, it, vi } from 'vitest';
+import { DESKTOP_AUDIO_RETRY_MS } from '../../../../shared/remoteDesktop';
+import { startDesktopCaptureHost } from '../captureHost';
+import { nativeCaptureStream } from '../nativeCaptureStream';
+
+vi.mock('../nativeCaptureStream', () => ({ nativeCaptureStream: vi.fn() }));
+const disposers: Array<() => void> = [];
+afterEach(() => {
+ disposers.splice(0).forEach((dispose) => dispose());
+ vi.unstubAllGlobals();
+ vi.useRealTimers();
+});
+const flush = async () => {
+ for (let i = 0; i < 30; i++) await Promise.resolve();
+};
+function media(audio = true) {
+ const video = { kind: 'video', stop: vi.fn() };
+ const sound = { kind: 'audio', stop: vi.fn() };
+ const tracks = audio ? [video, sound] : [video];
+ return {
+ video,
+ sound,
+ getTracks: () => tracks,
+ getVideoTracks: () => tracks.filter((track) => track.kind === 'video'),
+ getAudioTracks: () => tracks.filter((track) => track.kind === 'audio'),
+ addTrack: (track: typeof sound) => tracks.push(track),
+ };
+}
+function setup() {
+ vi.useFakeTimers();
+ const video = media(false);
+ const nativeStop = vi.fn();
+ vi.mocked(nativeCaptureStream).mockResolvedValue({
+ stream: video,
+ stop: nativeStop,
+ clear: vi.fn(),
+ } as any);
+ const capture = vi.fn().mockRejectedValue(new Error('NotAllowedError'));
+ vi.stubGlobal('navigator', { mediaDevices: { getDisplayMedia: capture } });
+ const peers: Peer[] = [];
+ class Peer {
+ remoteReady = false;
+ localDescription = { sdp: 'answer' };
+ iceGatheringState = 'complete';
+ close = vi.fn();
+ addTrack = vi.fn();
+ audio = {
+ direction: 'recvonly',
+ receiver: { track: { kind: 'audio' } },
+ sender: { setStreams: vi.fn(), replaceTrack: vi.fn(async (_track: unknown) => {}) },
+ };
+ constructor() {
+ peers.push(this);
+ }
+ getSenders() {
+ return [];
+ }
+ getTransceivers() {
+ expect(this.remoteReady).toBe(true);
+ return [this.audio];
+ }
+ async setRemoteDescription() {
+ this.remoteReady = true;
+ }
+ async setLocalDescription() {}
+ async createAnswer() {
+ return {};
+ }
+ }
+ vi.stubGlobal('RTCPeerConnection', Peer);
+ let command!: (value: any) => void;
+ const reply = vi.fn(async () => {});
+ const api = {
+ onCommand: (callback: typeof command) => {
+ command = callback;
+ return () => {};
+ },
+ registerHost: vi.fn(async () => {}),
+ reply,
+ };
+ disposers.push(startDesktopCaptureHost(api as any));
+ const offer = (audio = true, overlay = true) =>
+ command({
+ id: 'offer',
+ op: 'offer',
+ lease: 'lease',
+ sdp: 'offer',
+ attemptId: 'attempt',
+ sourceId: 'screen:1',
+ nativeCapture: true,
+ cursorOverlay: overlay,
+ settings: { audio, fps: 30 },
+ });
+ return { video, nativeStop, peers, capture, reply, offer, stop: () => command({ op: 'stop' }) };
+}
+
+it.each(['rejected', 'missing', 'without-overlay'])(
+ 'keeps video after %s audio and restores only its audio sender when permission becomes ready',
+ async (mode) => {
+ const h = setup();
+ const empty = media(false);
+ if (mode === 'missing') h.capture.mockResolvedValueOnce(empty);
+ h.offer(true, mode !== 'without-overlay');
+ await flush();
+ const peer = h.peers[0];
+ expect(h.reply).toHaveBeenCalledWith('offer', 'answer');
+ expect(peer.audio.direction).toBe('sendonly');
+ expect(peer.audio.sender.setStreams).toHaveBeenCalledWith(h.video);
+ expect(h.video.video.stop).not.toHaveBeenCalled();
+ const recovered = media();
+ h.capture.mockResolvedValueOnce(recovered);
+ await vi.advanceTimersByTimeAsync(DESKTOP_AUDIO_RETRY_MS[0]);
+ expect(peer.audio.sender.replaceTrack).toHaveBeenCalledWith(recovered.sound);
+ expect(h.peers).toHaveLength(1);
+ expect(peer.close).not.toHaveBeenCalled();
+ expect(recovered.video.stop).toHaveBeenCalledOnce();
+ expect(recovered.sound.stop).not.toHaveBeenCalled();
+ expect(h.video.getAudioTracks()).toEqual([recovered.sound]);
+ await vi.advanceTimersByTimeAsync(120_000);
+ expect(h.capture).toHaveBeenCalledTimes(2);
+ h.stop();
+ expect(recovered.sound.stop).toHaveBeenCalledOnce();
+ },
+);
+
+it('exhausts audio retries without closing video or restarting capture', async () => {
+ const h = setup();
+ h.offer();
+ await flush();
+ await vi.advanceTimersByTimeAsync(120_000);
+ expect(h.capture).toHaveBeenCalledTimes(1 + DESKTOP_AUDIO_RETRY_MS.length);
+ expect(h.peers).toHaveLength(1);
+ expect(h.peers[0].close).not.toHaveBeenCalled();
+ expect(h.video.video.stop).not.toHaveBeenCalled();
+ expect(h.nativeStop).not.toHaveBeenCalled();
+});
+
+it('waits for a timed-out permission request to settle before retrying and stops its late tracks', async () => {
+ const h = setup();
+ let finish!: (value: unknown) => void;
+ h.capture.mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ finish = resolve;
+ }),
+ );
+ h.offer();
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(h.reply).toHaveBeenCalledWith('offer', 'answer');
+ expect(h.capture).toHaveBeenCalledTimes(1);
+ const late = media();
+ finish(late);
+ await flush();
+ late.getTracks().forEach((track) => expect(track.stop).toHaveBeenCalledOnce());
+ const recovered = media();
+ h.capture.mockResolvedValueOnce(recovered);
+ await vi.advanceTimersByTimeAsync(DESKTOP_AUDIO_RETRY_MS[0]);
+ expect(h.peers[0].audio.sender.replaceTrack).toHaveBeenCalledWith(recovered.sound);
+});
+
+it.each(['capture', 'replaceTrack'] as const)(
+ 'fences late %s success after audio is disabled by a new offer',
+ async (phase) => {
+ const h = setup();
+ h.offer();
+ await flush();
+ let finish!: (value?: any) => void;
+ const recovered = media();
+ if (phase === 'capture')
+ h.capture.mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ finish = resolve;
+ }),
+ );
+ else {
+ h.capture.mockResolvedValueOnce(recovered);
+ h.peers[0].audio.sender.replaceTrack.mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ finish = resolve;
+ }),
+ );
+ }
+ await vi.advanceTimersByTimeAsync(DESKTOP_AUDIO_RETRY_MS[0]);
+ h.offer(false);
+ await flush();
+ finish(phase === 'capture' ? recovered : undefined);
+ await flush();
+ recovered.getTracks().forEach((track) => expect(track.stop).toHaveBeenCalledOnce());
+ await vi.advanceTimersByTimeAsync(120_000);
+ expect(h.capture).toHaveBeenCalledTimes(2);
+ expect(h.peers[1].audio.sender.replaceTrack).not.toHaveBeenCalled();
+ expect(h.peers[1].close).not.toHaveBeenCalled();
+ },
+);
+
+it('cancels scheduled recovery on stop and never attempts audio for an audio-off offer', async () => {
+ const h = setup();
+ h.offer();
+ await flush();
+ h.stop();
+ await vi.advanceTimersByTimeAsync(120_000);
+ expect(h.capture).toHaveBeenCalledTimes(1);
+ h.offer(false);
+ await flush();
+ await vi.advanceTimersByTimeAsync(120_000);
+ expect(h.capture).toHaveBeenCalledTimes(1);
+ expect(h.peers[1].audio.sender.setStreams).not.toHaveBeenCalled();
+});
+
+it('keeps retrying locally after replaceTrack rejects without dropping the video peer', async () => {
+ const h = setup();
+ h.offer();
+ await flush();
+ const failed = media();
+ h.capture.mockResolvedValueOnce(failed);
+ h.peers[0].audio.sender.replaceTrack.mockRejectedValueOnce(new Error('InvalidStateError'));
+ await vi.advanceTimersByTimeAsync(DESKTOP_AUDIO_RETRY_MS[0]);
+ failed.getTracks().forEach((track) => expect(track.stop).toHaveBeenCalledOnce());
+ const recovered = media();
+ h.capture.mockResolvedValueOnce(recovered);
+ await vi.advanceTimersByTimeAsync(DESKTOP_AUDIO_RETRY_MS[1]);
+ expect(h.peers[0].audio.sender.replaceTrack).toHaveBeenLastCalledWith(recovered.sound);
+ expect(h.peers[0].close).not.toHaveBeenCalled();
+});
diff --git a/apps/desktop/src/renderer/features/remote-desktop/captureHost.ts b/apps/desktop/src/renderer/features/remote-desktop/captureHost.ts
index 33c7c14f1b..80584060d9 100644
--- a/apps/desktop/src/renderer/features/remote-desktop/captureHost.ts
+++ b/apps/desktop/src/renderer/features/remote-desktop/captureHost.ts
@@ -7,7 +7,7 @@ import {
type DesktopInput,
type RemoteDesktopCursor,
} from '@cindy/device-link';
-import type { DesktopCaptureApi } from '../../../shared/remoteDesktop';
+import { DESKTOP_AUDIO_RETRY_MS, type DesktopCaptureApi } from '../../../shared/remoteDesktop';
import { nativeCaptureStream } from './nativeCaptureStream';
/** Runs exclusively in the isolated capture renderer; never import into the chat entry. */
@@ -28,11 +28,14 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
let gatheringTimer: ReturnType<typeof setTimeout> | undefined;
let finishGathering: (() => void) | undefined;
let exchanging = false;
+ let audioRetryTimer: ReturnType<typeof setTimeout> | undefined;
const stop = () => {
generation++;
exchanging = false;
clearTimeout(disconnectedTimer);
clearTimeout(gatheringTimer);
+ clearTimeout(audioRetryTimer);
+ audioRetryTimer = undefined;
finishGathering?.();
finishGathering = undefined;
disconnectedTimer = gatheringTimer = undefined;
@@ -124,6 +127,7 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
attemptId = command.attemptId;
void (async () => {
try {
+ let captureSettled: Promise<void> = Promise.resolve();
const capture = () =>
navigator.mediaDevices.getDisplayMedia({
audio: command.settings?.audio === true,
@@ -136,14 +140,21 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
let abandoned = false;
let timeout: ReturnType<typeof setTimeout> | undefined;
try {
+ const request = capture().then((value) => {
+ if (abandoned || current !== generation) {
+ value.getTracks().forEach((track) => track.stop());
+ throw new Error('DESKTOP_VIDEO_STOPPED');
+ }
+ return value;
+ });
+ // A timeout does not cancel getDisplayMedia or its permission prompt.
+ // Never overlap another request while the old one is still pending.
+ captureSettled = request.then(
+ () => {},
+ () => {},
+ );
return await Promise.race([
- capture().then((value) => {
- if (abandoned || current !== generation) {
- value.getTracks().forEach((track) => track.stop());
- throw new Error('DESKTOP_VIDEO_STOPPED');
- }
- return value;
- }),
+ request,
new Promise<never>((_, reject) => {
timeout = setTimeout(
() => {
@@ -208,10 +219,6 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
captured.getTracks().forEach((track) => track.stop());
return;
}
- if (command.settings?.audio && !captured.getAudioTracks().length) {
- captured.getTracks().forEach((track) => track.stop());
- throw new Error('DESKTOP_AUDIO_UNAVAILABLE');
- }
stream = captured;
const rtc = new RTCPeerConnection({
iceServers: command.iceServers ?? REMOTE_DESKTOP_ICE_SERVERS,
@@ -338,6 +345,19 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
};
stream.getTracks().forEach((track) => rtc.addTrack(track, captured));
await rtc.setRemoteDescription({ type: 'offer', sdp: command.sdp });
+ // Negotiate audio now even when permission is not ready. replaceTrack
+ // can fill this sender later without interrupting video or the data channel.
+ const audioTransceiver =
+ command.settings?.audio && !captured.getAudioTracks().length
+ ? rtc.getTransceivers().find((item) => item.receiver.track.kind === 'audio')
+ : undefined;
+ const audioSender = audioTransceiver?.sender;
+ if (audioTransceiver && audioSender) {
+ audioTransceiver.direction = 'sendonly';
+ // Both receivers must belong to the same stream, including when
+ // the viewer observes the initially silent audio track before video.
+ audioSender.setStreams(captured);
+ }
await rtc.setLocalDescription(await rtc.createAnswer());
for (const sender of rtc.getSenders()) {
if (sender.track?.kind !== 'video' || !command.settings) continue;
@@ -365,6 +385,36 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
};
});
if (current === generation) await api.reply(command.id, rtc.localDescription?.sdp ?? null);
+ const retryAudio = async (retry: number): Promise<void> => {
+ await captureSettled;
+ if (!audioSender || current !== generation || retry >= DESKTOP_AUDIO_RETRY_MS.length)
+ return;
+ audioRetryTimer = setTimeout(() => {
+ audioRetryTimer = undefined;
+ void (async () => {
+ let replacement: MediaStream | undefined;
+ let retained: MediaStreamTrack | undefined;
+ try {
+ if (current !== generation) return;
+ replacement = await boundedCapture();
+ const track = replacement.getAudioTracks()[0];
+ if (!track || current !== generation) return;
+ await audioSender.replaceTrack(track);
+ if (current !== generation) return;
+ captured.addTrack(track);
+ retained = track;
+ } catch {
+ // Audio recovery never tears down the working video connection.
+ } finally {
+ replacement?.getTracks().forEach((track) => {
+ if (track !== retained) track.stop();
+ });
+ if (!retained) void retryAudio(retry + 1);
+ }
+ })();
+ }, DESKTOP_AUDIO_RETRY_MS[retry]);
+ };
+ if (audioSender) void retryAudio(0);
} catch (error) {
if (current === generation) {
stop();
diff --git a/apps/desktop/src/shared/remoteDesktop.ts b/apps/desktop/src/shared/remoteDesktop.ts
index 205a051845..11e15d5243 100644
--- a/apps/desktop/src/shared/remoteDesktop.ts
+++ b/apps/desktop/src/shared/remoteDesktop.ts
@@ -8,6 +8,8 @@ import type {
RemoteDesktopIceReply,
DesktopIceServer,
} from '@cindy/device-link';
+// Initial capture plus three lease-scoped audio retries; also bounds Main's grant.
+export const DESKTOP_AUDIO_RETRY_MS = [3_000, 10_000, 30_000] as const;
export const DESKTOP_LOCAL = {
STATE: 'remote-desktop:state',
ENABLE: 'remote-desktop:enable',