Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

feat(remote-desktop): 手机画中画观看时电脑端自动按省流上限编码,回到全屏恢复原画质 (#5482)

Dash1e004643cfb3
diff --git a/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts b/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts
index 5f7c47c50b..026d7001e4 100644
--- a/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts
+++ b/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts
@@ -29,6 +29,7 @@ const h = vi.hoisted(() => ({
deps: null as any,
permissionDeps: null as any,
lease: 'lease',
+ background: false,
ready: false,
source: null as null | Promise<any[]>,
owner: null as any,
@@ -172,6 +173,9 @@ vi.mock('../controller', () => ({
hasLease(value: string) {
return value === h.lease;
}
+ isBackgroundViewing(value: string) {
+ return value === h.lease && h.background;
+ }
stop() {
h.stop();
h.deps.stopVideo();
@@ -255,6 +259,7 @@ beforeEach(() => {
vi.stubGlobal('process', { ...process, platform: 'darwin', getSystemVersion: () => '26.0' });
vi.useFakeTimers();
h.wayland = false;
+ h.background = false;
h.hyprland = false;
h.linuxInput = false;
h.linuxAudio = false;
@@ -1258,3 +1263,31 @@ it('revokes audio recovery with the lease and never grants it to an audio-off re
expect(callback).toHaveBeenLastCalledWith({});
expect(owner.dead).toBe(false);
});
+
+it('starts and retunes video for a background viewer without a new offer', async () => {
+ h.background = true;
+ const pending = h.deps.offer(
+ { lease: h.lease, display: { id: '1' } },
+ 'sdp',
+ { fps: 60, quality: 'auto', audio: false },
+ false,
+ 'attempt',
+ );
+ h.handlers.get(DESKTOP_LOCAL.REGISTER)(event());
+ await flush();
+ const offer = h.owner.send.mock.calls[0][1];
+ expect(offer).toMatchObject({ op: 'offer', lease: h.lease, background: true });
+ h.handlers.get(DESKTOP_LOCAL.REPLY)(event(), offer.id, 'answer');
+ await pending;
+ const owner = h.owner;
+ const sent = owner.send.mock.calls.length;
+ h.deps.videoBackground('stale', false);
+ expect(owner.send).toHaveBeenCalledTimes(sent);
+ h.deps.videoBackground(h.lease, false);
+ expect(owner.send.mock.calls.at(-1)[1]).toMatchObject({
+ op: 'background-viewing',
+ lease: h.lease,
+ background: false,
+ });
+ expect(h.windows).toHaveLength(1);
+});
diff --git a/apps/desktop/src/main/remote-desktop/__tests__/controller.test.ts b/apps/desktop/src/main/remote-desktop/__tests__/controller.test.ts
index 6e5d2d48e1..1922f5ef78 100644
--- a/apps/desktop/src/main/remote-desktop/__tests__/controller.test.ts
+++ b/apps/desktop/src/main/remote-desktop/__tests__/controller.test.ts
@@ -1435,3 +1435,45 @@ it('forwards viewerHidden for the bound lease without needing control', async ()
h.controller.request('phone', { op: 'viewerHidden', lease, hidden: false }),
).rejects.toThrow('DESKTOP_VIDEO_UNAVAILABLE');
});
+it('reports each background viewing change once, ending it when control returns', async () => {
+ const h = harness(),
+ { lease } = await h.start();
+ h.deps.videoBackground = vi.fn();
+ expect(h.controller.isBackgroundViewing(lease)).toBe(false);
+ await h.controller.request('phone', { op: 'presentation', lease, enabled: true });
+ await h.controller.request('phone', { op: 'presentation', lease, enabled: true });
+ expect(h.controller.isBackgroundViewing(lease)).toBe(true);
+ expect(h.controller.isBackgroundViewing('stale')).toBe(false);
+ await h.controller.request('phone', { op: 'control', lease, enabled: true });
+ expect(h.controller.isBackgroundViewing(lease)).toBe(false);
+ await h.controller.request('phone', { op: 'presentation', lease, enabled: true });
+ await h.controller.request('phone', { op: 'presentation', lease, enabled: false });
+ expect(vi.mocked(h.deps.videoBackground).mock.calls).toEqual([
+ [lease, true],
+ [lease, false],
+ [lease, true],
+ [lease, false],
+ ]);
+});
+it('keeps background viewing when a control request is refused as busy', async () => {
+ const h = harness(),
+ { lease } = await h.start();
+ let started!: () => void;
+ h.deps.startInput = vi.fn(
+ () =>
+ new Promise<void>((resolve) => {
+ started = resolve;
+ }),
+ );
+ h.deps.videoBackground = vi.fn();
+ const starting = h.controller.request('phone', { op: 'control', lease, enabled: true });
+ await Promise.resolve();
+ await h.controller.request('phone', { op: 'presentation', lease, enabled: true });
+ await expect(
+ h.controller.request('phone', { op: 'control', lease, enabled: true }),
+ ).rejects.toThrow('DESKTOP_INPUT_BUSY');
+ expect(h.controller.isBackgroundViewing(lease)).toBe(true);
+ expect(vi.mocked(h.deps.videoBackground).mock.calls.at(-1)).toEqual([lease, true]);
+ started();
+ await starting.catch(() => {});
+});
diff --git a/apps/desktop/src/main/remote-desktop/controller.ts b/apps/desktop/src/main/remote-desktop/controller.ts
index b8508c9ba7..2b1e97d3f6 100644
--- a/apps/desktop/src/main/remote-desktop/controller.ts
+++ b/apps/desktop/src/main/remote-desktop/controller.ts
@@ -112,6 +112,8 @@ export interface DesktopControllerDeps {
stopHostMute?(): Promise<void>;
/** Pauses or resumes sending the lease's current video stream; rejects if not applied. */
viewerHidden?(lease: string, hidden: boolean): Promise<void>;
+ /** The lease entered or left view-only background viewing (phone picture-in-picture). */
+ videoBackground?(lease: string, background: boolean): void;
changed(): void;
now?: () => number;
}
@@ -207,6 +209,17 @@ export class RemoteDesktopController {
this.tick();
return this.active?.lease === lease;
}
+ isBackgroundViewing(lease: string): boolean {
+ return this.active?.lease === lease && this.active.backgroundViewing === true;
+ }
+ private setBackgroundViewing(
+ active: NonNullable<RemoteDesktopController['active']>,
+ enabled: boolean,
+ ) {
+ if ((active.backgroundViewing === true) === enabled) return;
+ active.backgroundViewing = enabled;
+ this.deps.videoBackground?.(active.lease, enabled);
+ }
private now(): number {
return this.deps.now?.() ?? Date.now();
}
@@ -797,7 +810,7 @@ export class RemoteDesktopController {
active.expires = this.now() + REMOTE_DESKTOP_LEASE_MS;
return { controlling: active.controlling };
case 'presentation': {
- active.backgroundViewing = request.enabled;
+ this.setBackgroundViewing(active, request.enabled);
if (request.enabled) {
await this.revokeControl();
} else {
@@ -807,8 +820,8 @@ export class RemoteDesktopController {
}
case 'control': {
if (this.locking) throw new Error('DESKTOP_BUSY');
- if (request.enabled) active.backgroundViewing = false;
if (request.enabled && this.inputStarting) throw new Error('DESKTOP_INPUT_BUSY');
+ if (request.enabled) this.setBackgroundViewing(active, false);
if (!request.enabled) {
await this.revokeControl();
return { controlling: active.controlling };
diff --git a/apps/desktop/src/main/remote-desktop/index.ts b/apps/desktop/src/main/remote-desktop/index.ts
index 4aa30144df..c7f0bb18d4 100644
--- a/apps/desktop/src/main/remote-desktop/index.ts
+++ b/apps/desktop/src/main/remote-desktop/index.ts
@@ -261,6 +261,16 @@ async function setViewerHidden(lease: string, hidden: boolean): Promise<void> {
const applied = await requestHost({ id: randomUUID(), op: 'viewer-hidden', lease, hidden }, 2000);
if (applied !== true || lease !== videoLease) throw new Error('DESKTOP_VIDEO_UNAVAILABLE');
}
+/** Retunes the live encoder in place; a later offer reads the state from the lease. */
+function setVideoBackground(lease: string, background: boolean): void {
+ if (lease !== videoLease || !host || host.isDestroyed()) return;
+ host.send(DESKTOP_LOCAL.COMMAND, {
+ id: randomUUID(),
+ op: 'background-viewing',
+ lease,
+ background,
+ } satisfies DesktopHostCommand);
+}
function stopVideo(): void {
videoPaused = false;
offerGeneration++;
@@ -335,6 +345,7 @@ async function offer(
portalCapture: true,
sdp,
settings,
+ background: remoteDesktop.isBackgroundViewing(lease.lease),
attemptId,
iceServers,
},
@@ -429,6 +440,7 @@ async function offer(
lease: lease.lease,
sdp,
settings,
+ background: remoteDesktop.isBackgroundViewing(lease.lease),
attemptId,
iceServers,
},
@@ -735,6 +747,7 @@ export const remoteDesktop: RemoteDesktopController = new RemoteDesktopControlle
? linuxMute.set(false)
: systemAudioMuteGuard.restore('remote-desktop'),
viewerHidden: setViewerHidden,
+ videoBackground: setVideoBackground,
displayModes: readDesktopDisplayModes,
displayPresent: async (displayId) => {
if (nativeWayland()) {
diff --git a/apps/desktop/src/renderer/features/remote-desktop/__tests__/captureAudioRecovery.test.ts b/apps/desktop/src/renderer/features/remote-desktop/__tests__/captureAudioRecovery.test.ts
index 8b68d20457..83ede37cfd 100644
--- a/apps/desktop/src/renderer/features/remote-desktop/__tests__/captureAudioRecovery.test.ts
+++ b/apps/desktop/src/renderer/features/remote-desktop/__tests__/captureAudioRecovery.test.ts
@@ -112,6 +112,7 @@ function setup() {
quality?: 'auto' | 'saver' | 'hd',
fps: 30 | 60 = 30,
sdp = 'offer',
+ background = false,
) =>
command({
id: 'offer',
@@ -124,6 +125,7 @@ function setup() {
nativeAudio,
cursorOverlay: overlay,
settings: { audio, fps, ...(quality ? { quality } : {}) },
+ background,
});
const offerWithoutSettings = (sdp: string) =>
command({
@@ -146,6 +148,8 @@ function setup() {
swap: (lease = 'lease') => command({ id: 'swap', op: 'display-swap', lease }),
hide: (hidden: boolean, lease = 'lease') =>
command({ id: 'hide', op: 'viewer-hidden', lease, hidden }),
+ background: (background: boolean, lease = 'lease') =>
+ command({ id: 'background', op: 'background-viewing', lease, background }),
peers,
capture,
reply,
@@ -215,10 +219,19 @@ it('keeps full resolution while the screen is still only for tiers that ask for
expect.objectContaining({ degradationPreference: 'maintain-framerate' }),
);
+ // HD tracks motion too, since it may switch to the saver tier in the background,
+ // but in the foreground it keeps full resolution either way.
const hd = setup();
hd.offer(false, true, false, 'hd');
await flush();
- expect(vi.mocked(nativeCaptureStream).mock.calls.at(-1)?.[5]).toBeUndefined();
+ const hdMotion = vi.mocked(nativeCaptureStream).mock.calls.at(-1)?.[5];
+ expect(hdMotion).toBeTypeOf('function');
+ hdMotion!(false);
+ await flush();
+ hdMotion!(true);
+ await flush();
+ for (const [parameters] of hd.peers[0].video.setParameters.mock.calls)
+ expect(parameters).toMatchObject({ degradationPreference: 'maintain-resolution' });
});
it('passes the tier bandwidth floor to the viewer offer, leaving legacy offers untouched', async () => {
@@ -609,7 +622,8 @@ it('pauses and resumes the video encoder in place while the viewer is hidden', a
h.offer(false, true, false, 'auto');
await flush();
const [peer] = h.peers;
- let current: RTCRtpSendParameters = { encodings: [{}] } as RTCRtpSendParameters;
+ const tuned = { maxFramerate: 30, maxBitrate: 20_000_000 };
+ let current = { encodings: [{ ...tuned }] } as RTCRtpSendParameters;
peer.video.getParameters = () => structuredClone(current) as any;
const setParameters = vi.fn(async (next: any) => {
current = structuredClone(next);
@@ -622,7 +636,7 @@ it('pauses and resumes the video encoder in place while the viewer is hidden', a
expect(h.reply).toHaveBeenLastCalledWith('hide', false);
h.hide(true);
await flush();
- expect(current.encodings).toEqual([{ active: false }]);
+ expect(current.encodings).toEqual([{ ...tuned, active: false }]);
expect(h.reply).toHaveBeenLastCalledWith('hide', true);
// Motion updates keep the pause instead of overwriting it.
vi.mocked(nativeCaptureStream).mock.calls.at(-1)?.[5]?.(false);
@@ -636,7 +650,7 @@ it('pauses and resumes the video encoder in place while the viewer is hidden', a
expect(setParameters).toHaveBeenCalledTimes(2);
h.hide(false);
await flush();
- expect(current.encodings).toEqual([{ active: true }]);
+ expect(current.encodings).toEqual([{ ...tuned, active: true }]);
expect(h.peers).toHaveLength(1);
expect(peer.close).not.toHaveBeenCalled();
expect(h.nativeStop).not.toHaveBeenCalled();
@@ -672,3 +686,109 @@ it('reports a rejected encoder resume so the viewer rebuilds the video', async (
expect(h.reply).toHaveBeenLastCalledWith('hide', true);
expect(peer.close).not.toHaveBeenCalled();
});
+
+it('caps a background viewer at the saver tier in place and restores its own tier', async () => {
+ const h = setup();
+ h.offer(false, true, false, 'hd', 60);
+ await flush();
+ const [peer] = h.peers;
+ let current = structuredClone(peer.video.setParameters.mock.calls[0][0]) as RTCRtpSendParameters;
+ peer.video.getParameters = () => structuredClone(current) as any;
+ const setParameters = vi.fn(async (next: any) => {
+ current = structuredClone(next);
+ });
+ peer.video.setParameters = setParameters;
+
+ h.background(true, 'other-lease');
+ await flush();
+ expect(setParameters).not.toHaveBeenCalled();
+ h.background(true);
+ await flush();
+ expect(current).toEqual({
+ degradationPreference: 'maintain-framerate',
+ encodings: [{ maxFramerate: 30, maxBitrate: 2_000_000 }],
+ });
+ h.background(true);
+ await flush();
+ expect(setParameters).toHaveBeenCalledOnce();
+ h.background(false);
+ await flush();
+ expect(current).toEqual({
+ degradationPreference: 'maintain-resolution',
+ encodings: [{ maxFramerate: 60, maxBitrate: 20_000_000 }],
+ });
+ expect(h.peers).toHaveLength(1);
+ expect(peer.close).not.toHaveBeenCalled();
+ expect(h.nativeStop).not.toHaveBeenCalled();
+});
+
+it('starts a peer negotiated in the background at the saver ceilings, keeping its capture rate', async () => {
+ const h = setup();
+ h.offer(false, true, false, 'auto', 60, 'offer', true);
+ await flush();
+ expect(h.peers[0].video.setParameters).toHaveBeenCalledExactlyOnceWith({
+ degradationPreference: 'maintain-framerate',
+ encodings: [{ maxFramerate: 30, maxBitrate: 2_000_000 }],
+ });
+ // Returning to fullscreen only lifts the encoder ceilings; capture is not rebuilt.
+ expect(vi.mocked(nativeCaptureStream).mock.calls.at(-1)?.[4]).toBe(60);
+});
+
+it('applies a background change that arrives while the new peer is still being set up', async () => {
+ const h = setup();
+ h.offer(false, true, false, 'auto', 60);
+ h.background(true);
+ await flush();
+ expect(h.peers[0].video.setParameters).toHaveBeenLastCalledWith(
+ expect.objectContaining({ encodings: [{ maxFramerate: 30, maxBitrate: 2_000_000 }] }),
+ );
+});
+
+it('keeps a still HD screen sharp under the background saver ceilings', async () => {
+ const h = setup();
+ h.offer(false, true, false, 'hd', 60);
+ await flush();
+ const onMotion = vi.mocked(nativeCaptureStream).mock.calls.at(-1)?.[5];
+ const video = h.peers[0].video.setParameters;
+ h.background(true);
+ await flush();
+ expect(video).toHaveBeenLastCalledWith(
+ expect.objectContaining({ degradationPreference: 'maintain-framerate' }),
+ );
+ onMotion!(false);
+ await flush();
+ expect(video).toHaveBeenLastCalledWith({
+ degradationPreference: 'maintain-resolution',
+ encodings: [{ maxFramerate: 30, maxBitrate: 2_000_000 }],
+ });
+});
+
+it('converges a rejected background retune on the next heartbeat tick', async () => {
+ const h = setup();
+ h.offer(false, true, false, 'hd', 60);
+ await flush();
+ const [peer] = h.peers as any[];
+ let current = structuredClone(peer.video.setParameters.mock.calls[0][0]);
+ peer.video.getParameters = () => structuredClone(current);
+ let reject = true;
+ const setParameters = vi.fn(async (next: any) => {
+ if (reject) throw new Error('InvalidModificationError');
+ current = structuredClone(next);
+ });
+ peer.video.setParameters = setParameters;
+ vi.stubGlobal('crypto', { randomUUID: () => 'challenge' });
+ peer.ondatachannel({
+ channel: { label: 'input-v1', readyState: 'open', bufferedAmount: 0, send: vi.fn() },
+ });
+ h.background(true);
+ await flush();
+ expect(current.encodings).toEqual([{ maxFramerate: 60, maxBitrate: 20_000_000 }]);
+ reject = false;
+ await vi.advanceTimersByTimeAsync(2000);
+ expect(current.encodings).toEqual([{ maxFramerate: 30, maxBitrate: 2_000_000 }]);
+ // A matching encoder is left untouched on later ticks.
+ const calls = setParameters.mock.calls.length;
+ await vi.advanceTimersByTimeAsync(4000);
+ expect(setParameters).toHaveBeenCalledTimes(calls);
+ expect(peer.close).not.toHaveBeenCalled();
+});
diff --git a/apps/desktop/src/renderer/features/remote-desktop/captureHost.ts b/apps/desktop/src/renderer/features/remote-desktop/captureHost.ts
index db14cd8b61..37d36e3991 100644
--- a/apps/desktop/src/renderer/features/remote-desktop/captureHost.ts
+++ b/apps/desktop/src/renderer/features/remote-desktop/captureHost.ts
@@ -21,6 +21,7 @@ function channelRequestId(message: unknown): string | null {
}
import { DESKTOP_AUDIO_RETRY_MS, type DesktopCaptureApi } from '../../../shared/remoteDesktop';
import {
+ desktopEncoderLimits,
desktopVideoFramerate,
desktopVideoProfile,
withDesktopBitrateHints,
@@ -57,6 +58,10 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
// A hidden viewer pauses the video encoder in place; audio, input and the
// data channel keep running, and showing the viewer resumes the same stream.
let viewerHidden = false;
+ // A background viewer (phone picture-in-picture) gets the saver ceilings on
+ // the same stream; returning to fullscreen restores the negotiated tier.
+ let background = false;
+ let applyEncoder: (() => Promise<boolean>) | null = null;
// setParameters rejects stale parameters, so every sender update is queued.
let senderUpdates = Promise.resolve();
/** Resolves false when an encoder update was rejected. */
@@ -88,6 +93,8 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
const stop = () => {
generation++;
viewerHidden = false;
+ background = false;
+ applyEncoder = null;
exchanging = false;
clearTimeout(disconnectedTimer);
clearTimeout(gatheringTimer);
@@ -208,6 +215,12 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
void applyViewerHidden().then((applied) => api.reply(command.id, applied).catch(() => {}));
return;
}
+ if (command.op === 'background-viewing') {
+ if (command.lease !== activeLease) return;
+ background = command.background === true;
+ void applyEncoder?.();
+ return;
+ }
stop();
if (
command.op !== 'offer' ||
@@ -220,6 +233,7 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
const current = generation;
const lease = command.lease;
activeLease = lease;
+ background = command.background === true;
attemptId = command.attemptId;
void (async () => {
try {
@@ -229,16 +243,27 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
// Native capture reports whether the screen is moving; tiers that stay
// sharp when still trade frame rate for resolution only while still.
let moving = true;
- let tuned = false;
- const degradation = () => (moving ? profile.degradation : 'maintain-resolution');
+ // Below these ceilings WebRTC's congestion controller picks the rate;
+ // the tier decides whether resolution or frame rate gives way first.
+ const tune = () =>
+ updateVideoSenders((parameters) => {
+ const limits = desktopEncoderLimits(command.settings!, background);
+ const degradation =
+ moving || !limits.sharpWhenStill ? limits.degradation : 'maintain-resolution';
+ let changed = parameters.degradationPreference !== degradation;
+ parameters.degradationPreference = degradation;
+ for (const encoding of parameters.encodings) {
+ changed ||=
+ encoding.maxBitrate !== limits.maxBitrate ||
+ encoding.maxFramerate !== limits.maxFramerate;
+ encoding.maxBitrate = limits.maxBitrate;
+ encoding.maxFramerate = limits.maxFramerate;
+ }
+ return changed;
+ });
const onMotion = (next: boolean) => {
moving = next;
- if (!tuned || current !== generation) return;
- void updateVideoSenders((parameters) => {
- if (parameters.degradationPreference === degradation()) return false;
- parameters.degradationPreference = degradation();
- return true;
- });
+ if (current === generation) void applyEncoder?.();
};
const capture = () =>
navigator.mediaDevices.getDisplayMedia({
@@ -290,7 +315,8 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
if (current === generation) latestCursor = value;
},
command.cursorOverlay || command.continuousNativeCapture ? fps : 15,
- command.settings && profile.sharpWhenStill ? onMotion : undefined,
+ // Any tier may become the saver tier in the background, so motion is always tracked.
+ command.settings ? onMotion : undefined,
);
if (current !== generation) {
result.stop();
@@ -435,6 +461,9 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
let requests = 0;
let challenge = '';
heartbeat = setInterval(() => {
+ // Encoder ceilings follow state, not events: an update the encoder
+ // rejected converges here, and a matching encoder is left untouched.
+ if (current === generation) void applyEncoder?.();
// Keep one outstanding challenge until its reply arrives. The host
// lease bounds silence; replacing it here rejects valid slow pongs.
if (channel.readyState !== 'open' || current !== generation || challenge) return;
@@ -585,19 +614,11 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
};
}
await rtc.setLocalDescription(await rtc.createAnswer());
- for (const sender of rtc.getSenders()) {
- if (sender.track?.kind !== 'video' || !command.settings) continue;
- const parameters = sender.getParameters();
- if (!parameters.encodings?.length) continue;
- // Below these ceilings WebRTC's congestion controller picks the rate;
- // the tier decides whether resolution or frame rate gives way first.
- parameters.degradationPreference = degradation();
- tuned = true;
- for (const encoding of parameters.encodings) {
- encoding.maxFramerate = fps;
- encoding.maxBitrate = profile.maxBitrate;
- }
- await sender.setParameters(parameters);
+ if (current !== generation) throw new Error('DESKTOP_VIDEO_STOPPED');
+ if (command.settings) {
+ // Published before the first update, so a change arriving meanwhile queues after it.
+ applyEncoder = tune;
+ if (!(await tune())) throw new Error('DESKTOP_VIDEO_UNAVAILABLE');
}
// A pause that arrived while this peer was still being set up applies now.
if (viewerHidden) void applyViewerHidden();
diff --git a/apps/desktop/src/shared/__tests__/remoteDesktopQuality.test.ts b/apps/desktop/src/shared/__tests__/remoteDesktopQuality.test.ts
index adac1dd3a2..bccef51d5d 100644
--- a/apps/desktop/src/shared/__tests__/remoteDesktopQuality.test.ts
+++ b/apps/desktop/src/shared/__tests__/remoteDesktopQuality.test.ts
@@ -1,6 +1,7 @@
import { describe, expect, it } from 'vitest';
import {
DESKTOP_MOTION,
+ desktopEncoderLimits,
desktopFrameChange,
desktopVideoFramerate,
desktopVideoProfile,
@@ -40,6 +41,25 @@ describe('remote desktop quality tiers', () => {
}
});
+ it('caps a background viewer at the saver tier without changing its own choice', () => {
+ const hd = { fps: 60, quality: 'hd', audio: true } as const;
+ expect(desktopEncoderLimits(hd, false)).toEqual({
+ maxBitrate: 20_000_000,
+ maxFramerate: 60,
+ degradation: 'maintain-resolution',
+ sharpWhenStill: false,
+ });
+ expect(desktopEncoderLimits(hd, true)).toEqual({
+ maxBitrate: 2_000_000,
+ maxFramerate: 30,
+ degradation: 'maintain-framerate',
+ sharpWhenStill: true,
+ });
+ expect(hd.quality).toBe('hd');
+ const saver = { fps: 30, quality: 'saver', audio: false } as const;
+ expect(desktopEncoderLimits(saver, true)).toEqual(desktopEncoderLimits(saver, false));
+ });
+
it('adds bandwidth hints to every video media codec only', () => {
const profile = desktopVideoProfile({ fps: 30, quality: 'saver', audio: false });
const lines = withDesktopBitrateHints(offer, profile).split('\r\n');
diff --git a/apps/desktop/src/shared/remoteDesktop.ts b/apps/desktop/src/shared/remoteDesktop.ts
index cb01f96801..515cb479c4 100644
--- a/apps/desktop/src/shared/remoteDesktop.ts
+++ b/apps/desktop/src/shared/remoteDesktop.ts
@@ -38,6 +38,7 @@ export interface DesktopHostCommand {
| 'display-hold'
| 'display-swap'
| 'viewer-hidden'
+ | 'background-viewing'
| 'ice'
| 'prepare'
| 'frame';
@@ -54,6 +55,9 @@ export interface DesktopHostCommand {
cursorOverlay?: boolean;
/** viewer-hidden: stop sending video while the viewer is hidden. */
hidden?: boolean;
+ /** offer / background-viewing: the viewer is view-only in the background
+ * (phone picture-in-picture), so the encoder uses the saver ceilings. */
+ background?: boolean;
lease?: string;
sourceId?: string;
sdp?: string;
diff --git a/apps/desktop/src/shared/remoteDesktopQuality.ts b/apps/desktop/src/shared/remoteDesktopQuality.ts
index ee1242b1bb..8b41df9192 100644
--- a/apps/desktop/src/shared/remoteDesktopQuality.ts
+++ b/apps/desktop/src/shared/remoteDesktopQuality.ts
@@ -84,6 +84,22 @@ export function desktopVideoFramerate(settings?: RemoteDesktopVideoSettings): 30
return Math.min(settings?.fps ?? 30, desktopVideoProfile(settings).maxFramerate) as 30 | 60;
}
+/** Live encoder ceilings. A background viewer (phone picture-in-picture) only
+ * shows a small window, so it gets the saver tier until it returns to fullscreen;
+ * the viewer's own choice is untouched and applies again once it is back. */
+export function desktopEncoderLimits(
+ settings: RemoteDesktopVideoSettings,
+ background: boolean,
+): Pick<DesktopVideoProfile, 'maxBitrate' | 'degradation' | 'sharpWhenStill'> & {
+ maxFramerate: 30 | 60;
+} {
+ const tier: RemoteDesktopVideoSettings = background
+ ? { ...settings, quality: 'saver' }
+ : settings;
+ const { maxBitrate, degradation, sharpWhenStill } = desktopVideoProfile(tier);
+ return { maxBitrate, degradation, sharpWhenStill, maxFramerate: desktopVideoFramerate(tier) };
+}
+
const BITRATE_HINTS = ['x-google-start-bitrate', 'x-google-min-bitrate', 'x-google-max-bitrate'];
const MEDIA_CODECS = /^(H264|VP8|VP9|AV1|H265)$/i;
diff --git a/docs/dev-rules/protocol-compatibility.md b/docs/dev-rules/protocol-compatibility.md
index bc6b8deec5..6451cb34d6 100644
--- a/docs/dev-rules/protocol-compatibility.md
+++ b/docs/dev-rules/protocol-compatibility.md
@@ -179,7 +179,9 @@ Mobile 未新增卡片入口。服务端无需改动。
`offer.settings` 的画质由码率改为档位 `quality: "auto" | "saver" | "hd"`(自动/省流/高清)。
控制端只表达意图,具体的码率上限、降级取舍(`auto`/`saver` 先降分辨率保帧数,`hd` 锁分辨率
降帧数)、截屏分辨率与 JPEG 预算由被控端 `apps/desktop/src/shared/remoteDesktopQuality.ts`
-决定,调整数值无需两端同时发版。
+决定,调整数值无需两端同时发版。控制端处于后台观看(`presentation` 已开启,如手机画中画)
+期间,被控端临时按 `saver` 档的码率/帧率上限编码,回到前台即恢复所选档位;这是被控端本地
+行为,控制端发送的档位不变,也不新增协议字段。
新控制端经 `remoteDesktopVideoSettingsWire` 同时发送档位与旧 `bitrate`(auto→0、saver→2M、
hd→20M):旧被控端只校验 `bitrate` 并忽略 `quality`,无需新增能力声明。新被控端优先读取
diff --git a/docs/remote-desktop.md b/docs/remote-desktop.md
index f5ef9f8fb5..4ec462e97d 100644
--- a/docs/remote-desktop.md
+++ b/docs/remote-desktop.md
@@ -802,6 +802,11 @@ first frame, with host authorization completed in parallel during Home entry.
Entering releases control. A capture-renderer challenge/pong heartbeat renews
only a view-only lease after host authorization while the viewer reports actual
system PiP presentation.
+While background viewing is active the host caps the live video sender at the
+saver tier (2 Mbps, 30 fps, frame rate kept while moving) on the same peer, and
+lifts the cap as soon as the viewer returns to fullscreen or regains control; the
+viewer's own quality choice is not changed. A peer negotiated during background
+viewing starts at the same cap. This is host-local and needs no new capability.
Closing PiP, closing WebRTC, local disconnect, revocation and the ordinary finite
lease timeout all terminate background viewing. This does not grant indefinite
background control or extend the lifetime of unrelated device links.