Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

feat(remote-desktop): 画质改为自动/省流/高清三档并修复静止画面模糊 (#5384)

Dashcaf72d87634b
diff --git a/apps/desktop/native/remote-desktop/macos-capture.m b/apps/desktop/native/remote-desktop/macos-capture.m
index ea1742bb01..cc4e5f90b3 100644
--- a/apps/desktop/native/remote-desktop/macos-capture.m
+++ b/apps/desktop/native/remote-desktop/macos-capture.m
@@ -103,13 +103,17 @@ static NSDictionary *readCursor(CGDirectDisplayID display) {
int main(int argc, const char *argv[]) {
@autoreleasepool {
- BOOL overlay = (argc == 5 || argc == 6) && strcmp(argv[2], "cursor-overlay") == 0;
+ // Overlay: <display> cursor-overlay <fps> <quality> <physical max edge, 0 = logical> <max JPEG bytes> [excluded]
+ BOOL overlay = (argc == 7 || argc == 8) && strcmp(argv[2], "cursor-overlay") == 0;
if (argc != 2 && argc != 3 && !overlay) return 2;
- NSString *excluded = (overlay && argc == 6) || (!overlay && argc == 3)
+ NSString *excluded = (overlay && argc == 8) || (!overlay && argc == 3)
? [NSString stringWithUTF8String:argv[argc - 1]] : @"";
int fps = overlay && strcmp(argv[3], "60") == 0 ? 60 : (overlay ? 30 : 15);
double quality = overlay ? atof(argv[4]) : 0.55;
if (quality < 0.1 || quality > 1) return 2;
+ long physicalEdge = overlay ? atol(argv[5]) : 0;
+ long frameLimit = overlay ? atol(argv[6]) : 180000;
+ if (physicalEdge < 0 || physicalEdge > 8192 || frameLimit < 100000 || frameLimit > 4000000) return 2;
char *end = NULL;
unsigned long value = strtoul(argv[1], &end, 10);
if (!*argv[1] || *end || value > UINT32_MAX || !CGDisplayIsOnline((uint32_t)value)) return 2;
@@ -127,13 +131,27 @@ int main(int argc, const char *argv[]) {
CGDirectDisplayID display = (uint32_t)value;
size_t width = CGDisplayPixelsWide(display), height = CGDisplayPixelsHigh(display);
if (!width || !height) return 2;
+ // CGDisplayPixelsWide reports logical points on HiDPI modes. Higher tiers
+ // capture backing pixels instead, bounded by the requested long edge.
+ double edgeLimit = overlay ? 4096.0 : 1280.0;
+ if (physicalEdge) {
+ CGDisplayModeRef mode = CGDisplayCopyDisplayMode(display);
+ if (mode) {
+ size_t pixelWidth = CGDisplayModeGetPixelWidth(mode), pixelHeight = CGDisplayModeGetPixelHeight(mode);
+ CGDisplayModeRelease(mode);
+ if (pixelWidth > width && pixelHeight > height) { width = pixelWidth; height = pixelHeight; }
+ }
+ // Never below what the logical capture would have produced.
+ double logicalEdge = (double)MAX(CGDisplayPixelsWide(display), CGDisplayPixelsHigh(display));
+ edgeLimit = MAX((double)physicalEdge, MIN(edgeLimit, logicalEdge));
+ }
// This is a negotiated capture mode, not a snapshot of cursor availability.
// A temporarily hidden/unreadable cursor must not permanently bake the
// cursor into video or stop subsequent shape/visibility polling.
BOOL separateCursor = overlay;
// Preserve full selected display resolution for the cursor-overlay path.
// The legacy compatibility path retains its original inexpensive bounds.
- double scale = MIN(1.0, (overlay ? 4096.0 : 1280.0) / MAX(width, height));
+ double scale = MIN(1.0, edgeLimit / MAX(width, height));
width = MAX(1, (size_t)(width * scale)); height = MAX(1, (size_t)(height * scale));
dispatch_queue_t queue = dispatch_queue_create("cindy.desktop.capture", DISPATCH_QUEUE_SERIAL);
CIContext *context = [CIContext contextWithOptions:@{ kCIContextCacheIntermediates: @NO }];
@@ -172,7 +190,7 @@ int main(int argc, const char *argv[]) {
CIImage *image = [CIImage imageWithIOSurface:surface];
NSData *jpeg = [context JPEGRepresentationOfImage:image colorSpace:color
options:@{(__bridge NSString *)kCGImageDestinationLossyCompressionQuality: @(quality)}];
- NSUInteger limit = overlay ? 1000000 : 180000;
+ NSUInteger limit = (NSUInteger)frameLimit;
if (overlay) {
for (NSNumber *q in @[@0.45, @0.25, @0.1]) {
if (jpeg.length <= limit) break;
diff --git a/apps/desktop/scripts/remote-desktop-viewer-smoke.mjs b/apps/desktop/scripts/remote-desktop-viewer-smoke.mjs
index 4e63f1ee7f..c824a8eb21 100644
--- a/apps/desktop/scripts/remote-desktop-viewer-smoke.mjs
+++ b/apps/desktop/scripts/remote-desktop-viewer-smoke.mjs
@@ -311,6 +311,22 @@ try {
),
'menu keyboard navigation never reaches the remote computer',
);
+ // Fixed short choices use the shared segmented control; reselecting the current
+ // option keeps the fixture's negotiated settings unchanged.
+ const checkSegmented = async (name, option, screenshot) => {
+ const group = settings.getByRole('radiogroup', { name, exact: true });
+ const choice = group.getByRole('radio', { name: option, exact: true });
+ await choice.click();
+ assert((await choice.getAttribute('aria-checked')) === 'true', `${name} keeps ${option}`);
+ const bounds = await group.boundingBox();
+ const viewport = viewer.viewportSize();
+ assert(bounds && viewport, `${name} is visible`);
+ assert(
+ bounds.x >= 0 && bounds.x + bounds.width <= viewport.width,
+ `${name} stays within horizontal bounds`,
+ );
+ await viewer.screenshot({ animations: 'disabled', path: path.join(artifacts, screenshot) });
+ };
const checkExitDialog = async (theme) => {
const labels = await viewer.evaluate(async () => {
const catalog = (await import('/i18n/index.ts')).default;
@@ -401,7 +417,7 @@ try {
animations: 'disabled',
path: path.join(artifacts, 'settings-zh-dark.png'),
});
- await checkSelect(settings.getByRole('combobox').first(), 'fps-select-zh-dark.png');
+ await checkSegmented('帧率', '30 fps', 'fps-segmented-zh-dark.png');
await viewer.evaluate(async () => {
const { themeService } = await import('/themes/theme-service.ts');
const { cindyLight } = await import('/themes/builtin/cindy-light.ts');
@@ -411,7 +427,7 @@ try {
animations: 'disabled',
path: path.join(artifacts, 'settings-zh-light.png'),
});
- await checkSelect(settings.getByRole('combobox').nth(1), 'quality-select-zh-light.png');
+ await checkSegmented('画质', '自动', 'quality-segmented-zh-light.png');
const desktopButton = settings.getByRole('button', { name: '桌面', exact: true });
assert(await desktopButton.isEnabled(), 'desktop viewer opens with control');
await viewer.waitForFunction(() => {
@@ -451,7 +467,7 @@ try {
toolbar && settingsBounds && settingsBounds.y >= toolbar.y + toolbar.height,
'settings follow the wrapped toolbar',
);
- await checkSelect(settings.getByRole('combobox').first(), 'fps-select-narrow.png');
+ await checkSegmented('帧率', '30 fps', 'fps-segmented-narrow.png');
await viewer.setViewportSize({ width: 720, height: 420 });
if (controllerPlatform === 'darwin') {
await viewer.getByRole('button', { name: '关闭', exact: true }).click();
diff --git a/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts b/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts
index 2f1044e8d7..f5ee86f8c2 100644
--- a/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts
+++ b/apps/desktop/src/main/remote-desktop/__tests__/captureBoundary.test.ts
@@ -1062,7 +1062,7 @@ it.each([true, false])(
const pending = h.deps.offer(
{ lease: h.lease, display: { id: '1' } },
'sdp',
- { audio: true, fps: 30, bitrate: 0 },
+ { audio: true, fps: 30, quality: 'auto' },
true,
'attempt',
);
@@ -1108,7 +1108,7 @@ it('revokes audio recovery with the lease and never grants it to an audio-off re
const pending = h.deps.offer(
{ lease: h.lease, display: { id: '1' } },
'sdp',
- { audio: true, fps: 30, bitrate: 0 },
+ { audio: true, fps: 30, quality: 'auto' },
true,
'attempt',
);
diff --git a/apps/desktop/src/main/remote-desktop/__tests__/linuxNativeCapture.test.ts b/apps/desktop/src/main/remote-desktop/__tests__/linuxNativeCapture.test.ts
index eb75128da4..48bdc7ac33 100644
--- a/apps/desktop/src/main/remote-desktop/__tests__/linuxNativeCapture.test.ts
+++ b/apps/desktop/src/main/remote-desktop/__tests__/linuxNativeCapture.test.ts
@@ -33,7 +33,7 @@ it('reuses one child for large bounded frames and releases it on cancellation',
const { child, owner } = setup();
const first = owner.frame('wayland-portal');
await vi.waitFor(() => expect(h.spawn).toHaveBeenCalledOnce());
- expect(h.spawn).toHaveBeenCalledWith('/capture', ['video', '1', '65'], { stdio: 'pipe' });
+ expect(h.spawn).toHaveBeenCalledWith('/capture', ['video', '1', '80'], { stdio: 'pipe' });
await expect(owner.frame('wayland-portal')).resolves.toBeNull();
const jpeg = 'a'.repeat(400000);
child.stdout.write(jpeg + '\n');
@@ -86,11 +86,11 @@ it('parses a bounded Linux cursor frame and applies requested encoding quality',
const { child, owner } = setup();
const pending = owner.frame('wayland-portal', true, {
fps: 60,
- bitrate: 20_000_000,
+ quality: 'hd',
audio: false,
});
await vi.waitFor(() => expect(h.spawn).toHaveBeenCalledOnce());
- expect(h.spawn).toHaveBeenCalledWith('/capture', ['cursor-overlay', '1', '95'], {
+ expect(h.spawn).toHaveBeenCalledWith('/capture', ['cursor-overlay', '1', '90'], {
stdio: 'pipe',
});
const frame = { jpeg: 'anBlZw==', cursor: null };
diff --git a/apps/desktop/src/main/remote-desktop/__tests__/macNativeCapture.test.ts b/apps/desktop/src/main/remote-desktop/__tests__/macNativeCapture.test.ts
new file mode 100644
index 0000000000..b8300f4b40
--- /dev/null
+++ b/apps/desktop/src/main/remote-desktop/__tests__/macNativeCapture.test.ts
@@ -0,0 +1,62 @@
+import { EventEmitter } from 'node:events';
+import { PassThrough } from 'node:stream';
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+const h = vi.hoisted(() => ({ spawn: vi.fn() }));
+vi.mock('node:child_process', () => ({ spawn: h.spawn, execFile: vi.fn() }));
+vi.mock('electron', () => ({
+ app: { isPackaged: true },
+ nativeImage: {},
+ screen: { getAllDisplays: () => [{ id: 1, scaleFactor: 2 }] },
+}));
+vi.mock('../windowsHost', () => ({ openWindowsDesktopConnection: vi.fn() }));
+import { NativeDesktopCapture } from '../nativeCapture';
+
+const owners: NativeDesktopCapture[] = [];
+beforeEach(() => {
+ vi.stubGlobal('process', { ...process, platform: 'darwin', resourcesPath: '/res' });
+ h.spawn.mockReset();
+});
+afterEach(() => {
+ owners.splice(0).forEach((o) => o.stop());
+ vi.unstubAllGlobals();
+});
+function setup() {
+ const child = Object.assign(new EventEmitter(), {
+ stdin: new PassThrough(),
+ stdout: new PassThrough(),
+ stderr: new PassThrough(),
+ kill: vi.fn(),
+ });
+ h.spawn.mockReturnValue(child);
+ const owner = new NativeDesktopCapture();
+ owners.push(owner);
+ return { child, owner };
+}
+
+it.each([
+ ['auto', 60, ['60', '0.8', '2560', '1500000']],
+ ['saver', 60, ['30', '0.65', '0', '1000000']],
+ ['hd', 30, ['30', '0.9', '3840', '3000000']],
+] as const)('passes the %s tier capture budget to the macOS helper', async (quality, fps, args) => {
+ const { owner } = setup();
+ void owner.frame('1', true, { fps, quality, audio: false });
+ await vi.waitFor(() => expect(h.spawn).toHaveBeenCalledOnce());
+ expect(h.spawn.mock.calls[0][1]).toEqual(['1', 'cursor-overlay', ...args]);
+});
+
+it('accepts frames up to the tier budget and stops capture beyond it', async () => {
+ const frame = { jpeg: 'a'.repeat(2_666_668), cursor: null };
+ const hd = setup();
+ const accepted = hd.owner.frame('1', true, { fps: 30, quality: 'hd', audio: false });
+ await vi.waitFor(() => expect(h.spawn).toHaveBeenCalledOnce());
+ hd.child.stdout.write(JSON.stringify(frame) + '\n');
+ await expect(accepted).resolves.toEqual(frame);
+
+ const auto = setup();
+ const rejected = auto.owner.frame('1', true, { fps: 30, quality: 'auto', audio: false });
+ await vi.waitFor(() => expect(h.spawn).toHaveBeenCalledTimes(2));
+ auto.child.stdout.write(JSON.stringify(frame) + '\n');
+ await vi.waitFor(() => expect(auto.child.kill).toHaveBeenCalledOnce());
+ auto.owner.stop();
+ await expect(rejected).resolves.toBeNull();
+});
diff --git a/apps/desktop/src/main/remote-desktop/__tests__/nativeCapture.test.ts b/apps/desktop/src/main/remote-desktop/__tests__/nativeCapture.test.ts
index 6f7f18c537..28c437d0a9 100644
--- a/apps/desktop/src/main/remote-desktop/__tests__/nativeCapture.test.ts
+++ b/apps/desktop/src/main/remote-desktop/__tests__/nativeCapture.test.ts
@@ -36,7 +36,7 @@ afterEach(() => {
it('negotiates only the overlay path and preserves legacy init and response', async () => {
await expect(
- capture.frame('1', true, { fps: 60, bitrate: 8_000_000, audio: false }),
+ capture.frame('1', true, { fps: 60, quality: 'auto', audio: false }),
).resolves.toEqual({ jpeg: 'anBlZw==', cursor: null });
expect(openWindowsDesktopConnection).toHaveBeenLastCalledWith({
mode: 'capture',
@@ -60,7 +60,7 @@ it('reuses capture until video quality changes, then closes the old connection',
await capture.frame('1', true);
await capture.frame('1', true);
expect(connections).toHaveLength(1);
- await capture.frame('1', true, { fps: 30, bitrate: 20_000_000, audio: true });
+ await capture.frame('1', true, { fps: 30, quality: 'hd', audio: true });
expect(connections).toHaveLength(2);
expect(connections[0].close).toHaveBeenCalledOnce();
});
diff --git a/apps/desktop/src/main/remote-desktop/hyprlandCapture.ts b/apps/desktop/src/main/remote-desktop/hyprlandCapture.ts
index a4b8aa1e28..7e3a5af3f8 100644
--- a/apps/desktop/src/main/remote-desktop/hyprlandCapture.ts
+++ b/apps/desktop/src/main/remote-desktop/hyprlandCapture.ts
@@ -3,6 +3,7 @@ import { execFile } from 'node:child_process';
import { accessSync, constants } from 'node:fs';
import { WAYLAND_DISPLAY_ID, isWaylandDesktop } from './waylandCapture';
import type { RemoteDesktopCursorFrame, RemoteDesktopVideoSettings } from '@cindy/device-link';
+import { desktopVideoProfile } from '../../shared/remoteDesktopQuality';
import { linuxMonitor } from './linuxDesktop';
// Use the distribution's helper, never a command supplied by a remote peer or PATH.
@@ -59,7 +60,7 @@ export class HyprlandCapture {
'-t',
'jpeg',
'-q',
- settings?.bitrate === 20_000_000 ? '95' : settings?.bitrate === 8_000_000 ? '80' : '65',
+ String(Math.round(desktopVideoProfile(settings).jpegQuality * 100)),
'-s',
'1',
...(output ? ['-o', output.name] : []),
diff --git a/apps/desktop/src/main/remote-desktop/nativeCapture.ts b/apps/desktop/src/main/remote-desktop/nativeCapture.ts
index 841014ee52..15758b85ae 100644
--- a/apps/desktop/src/main/remote-desktop/nativeCapture.ts
+++ b/apps/desktop/src/main/remote-desktop/nativeCapture.ts
@@ -9,6 +9,7 @@ import { createHash } from 'node:crypto';
import fs from 'node:fs/promises';
import path from 'node:path';
import { promisify } from 'node:util';
+import { desktopVideoFramerate, desktopVideoProfile } from '../../shared/remoteDesktopQuality';
import { openWindowsDesktopConnection, type WindowsDesktopConnection } from './windowsHost';
import { decodeWindowsCursorFrame } from './windowsCursorFrame';
@@ -119,9 +120,11 @@ export class NativeDesktopCapture {
overlay = false,
settings?: RemoteDesktopVideoSettings,
): Promise<string | RemoteDesktopCursorFrame | null> {
+ const profile = desktopVideoProfile(settings);
+ const fps = desktopVideoFramerate(settings);
if (process.platform === 'win32') {
if (this.busy) return null;
- const config = overlay ? 'overlay:' + (settings?.bitrate ?? 0) : '';
+ const config = overlay ? 'overlay:' + profile.windowsBitrate : '';
if (this.windows && (this.display !== display || this.config !== config)) this.stop();
const selected = screen.getAllDisplays().find((item) => String(item.id) === display);
if (!selected) return null;
@@ -133,7 +136,7 @@ export class NativeDesktopCapture {
const connection = await openWindowsDesktopConnection({
mode: 'capture',
rect: [bounds.x, bounds.y, bounds.width, bounds.height],
- ...(overlay ? { cursorOverlay: true, bitrate: settings?.bitrate ?? 0 } : {}),
+ ...(overlay ? { cursorOverlay: true, bitrate: profile.windowsBitrate } : {}),
});
if (generation !== this.generation) {
connection.close();
@@ -164,7 +167,7 @@ export class NativeDesktopCapture {
}
const config =
overlay || process.platform === 'linux'
- ? [overlay, settings?.fps ?? 30, settings?.bitrate ?? 0].join(':')
+ ? [overlay, fps, settings?.quality ?? 'auto'].join(':')
: '';
if (this.child && this.config !== config) this.stop();
const linux = process.platform === 'linux';
@@ -183,8 +186,7 @@ export class NativeDesktopCapture {
const output = linux && display !== WAYLAND_DISPLAY_ID ? await linuxMonitor(display) : null;
const executable = await binary();
if (generation !== this.generation) return null;
- const quality =
- settings?.bitrate === 20_000_000 ? 0.95 : settings?.bitrate === 8_000_000 ? 0.8 : 0.65;
+ const quality = profile.jpegQuality;
const args = linux
? [
overlay ? 'cursor-overlay' : 'video',
@@ -193,7 +195,14 @@ export class NativeDesktopCapture {
...(output ? [output.name] : []),
]
: overlay
- ? [display, 'cursor-overlay', String(settings?.fps ?? 30), String(quality)]
+ ? [
+ display,
+ 'cursor-overlay',
+ String(fps),
+ String(quality),
+ String(profile.physicalMaxEdge),
+ String(profile.maxFrameBytes),
+ ]
: [display];
if (!linux && this.excludedWindows) args.push(this.excludedWindows);
const child = spawn(executable, args, { stdio: 'pipe' });
@@ -218,6 +227,11 @@ export class NativeDesktopCapture {
});
}
const child = this.child;
+ // macOS overlay frames follow the tier's byte budget; the base64 frame
+ // plus the bounded cursor PNG and JSON keep the historical 1.5 MB margin.
+ const frameBytes = overlay && !linux ? profile.maxFrameBytes : 1_000_000;
+ const jpegLimit = Math.ceil(frameBytes / 3) * 4;
+ const textLimit = jpegLimit + 166_664;
return await new Promise<string | RemoteDesktopCursorFrame | null>((resolve, reject) => {
let text = '';
const finish = (frame: string | RemoteDesktopCursorFrame | null, error?: Error) => {
@@ -231,7 +245,7 @@ export class NativeDesktopCapture {
};
const receive = (chunk: Buffer) => {
text += chunk.toString('ascii');
- if (text.length > (overlay || linux ? 1_500_000 : 240_001)) {
+ if (text.length > (overlay || linux ? textLimit : 240_001)) {
this.stop();
return;
}
@@ -241,7 +255,7 @@ export class NativeDesktopCapture {
const value = JSON.parse(text) as RemoteDesktopCursorFrame;
if (
typeof value.jpeg !== 'string' ||
- value.jpeg.length > 1_333_336 ||
+ value.jpeg.length > jpegLimit ||
!/^[A-Za-z0-9+/]+={0,2}$/.test(value.jpeg) ||
(value.cursor !== null && !isRemoteDesktopCursor(value.cursor))
)
diff --git a/apps/desktop/src/renderer/features/remote-desktop/RemoteDesktopViewerWindow.tsx b/apps/desktop/src/renderer/features/remote-desktop/RemoteDesktopViewerWindow.tsx
index d6858993af..d56b2e1150 100644
--- a/apps/desktop/src/renderer/features/remote-desktop/RemoteDesktopViewerWindow.tsx
+++ b/apps/desktop/src/renderer/features/remote-desktop/RemoteDesktopViewerWindow.tsx
@@ -11,7 +11,11 @@ import {
X,
Monitor,
} from 'lucide-react';
-import type { RemoteDesktopDisplayMode } from '@cindy/device-link';
+import {
+ REMOTE_DESKTOP_VIDEO_QUALITIES,
+ type RemoteDesktopDisplayMode,
+ type RemoteDesktopVideoQuality,
+} from '@cindy/device-link';
import { WindowControls } from '@/components/title-bar/WindowControls';
import { useMacFullscreen } from '@/hooks/useMacFullscreen';
import i18n from '@/i18n';
@@ -19,11 +23,18 @@ import { DesktopViewerController, type ViewerSnapshot } from './viewerController
import { Select } from '@/components/ui/select';
import { Button } from '@/components/ui/button';
import { FormField } from '@/components/ui/form-field';
+import { SegmentedControl } from '@/components/ui/segmented-control';
import { ConfirmDialog } from '@/components/ui/confirm-dialog';
import { Switch } from '@/components/ui/switch';
import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover';
import { Tip } from '@/components/ui/tooltip';
+const QUALITY_LABELS = {
+ auto: 'remoteDesktop.viewer.automatic',
+ saver: 'remoteDesktop.viewer.saver',
+ hd: 'remoteDesktop.viewer.hd',
+} as const satisfies Record<RemoteDesktopVideoQuality, string>;
+
/** A clean, standalone remote desktop surface. No App, router, agent or task providers. */
export function RemoteDesktopViewerWindow() {
const { t } = useTranslation();
@@ -335,12 +346,11 @@ export function RemoteDesktopViewerWindow() {
{state?.caps?.videoSettings && (
<>
<FormField label={t('remoteDesktop.viewer.fps')} className="remote-viewer-field">
- {({ id }) => (
- <Select
- id={id}
- className="w-full"
- label={t('remoteDesktop.viewer.fps')}
- value={String(state.settings.fps)}
+ {() => (
+ <SegmentedControl
+ fullWidth
+ aria-label={t('remoteDesktop.viewer.fps')}
+ value={String(state.settings.fps) as '30' | '60'}
options={[
{ value: '30', label: '30 fps' },
{ value: '60', label: '60 fps' },
@@ -348,32 +358,24 @@ export function RemoteDesktopViewerWindow() {
onValueChange={(value) =>
controller.current?.settings({ fps: Number(value) as 30 | 60 })
}
- onOpenChange={onSelectOpenChange}
/>
)}
</FormField>
<FormField
label={t('remoteDesktop.viewer.quality')}
+ hint={t('remoteDesktop.viewer.qualityHint')}
className="remote-viewer-field"
>
- {({ id }) => (
- <Select
- id={id}
- className="w-full"
- label={t('remoteDesktop.viewer.quality')}
- value={String(state.settings.bitrate)}
- options={[0, 2000000, 8000000, 20000000].map((value, index) => ({
- value: String(value),
- label: t(
- `remoteDesktop.viewer.${['automatic', 'smooth', 'balanced', 'clear'][index]}`,
- ),
+ {() => (
+ <SegmentedControl
+ fullWidth
+ aria-label={t('remoteDesktop.viewer.quality')}
+ value={state.settings.quality}
+ options={REMOTE_DESKTOP_VIDEO_QUALITIES.map((value) => ({
+ value,
+ label: t(QUALITY_LABELS[value]),
}))}
- onValueChange={(value) =>
- controller.current?.settings({
- bitrate: Number(value) as 0 | 2000000 | 8000000 | 20000000,
- })
- }
- onOpenChange={onSelectOpenChange}
+ onValueChange={(quality) => controller.current?.settings({ quality })}
/>
)}
</FormField>
diff --git a/apps/desktop/src/renderer/features/remote-desktop/__tests__/captureAudioRecovery.test.ts b/apps/desktop/src/renderer/features/remote-desktop/__tests__/captureAudioRecovery.test.ts
index 65e2991bf5..ded5345f93 100644
--- a/apps/desktop/src/renderer/features/remote-desktop/__tests__/captureAudioRecovery.test.ts
+++ b/apps/desktop/src/renderer/features/remote-desktop/__tests__/captureAudioRecovery.test.ts
@@ -72,8 +72,10 @@ function setup() {
expect(this.remoteReady).toBe(true);
return [this.audio];
}
- async setRemoteDescription() {
+ remote: { sdp?: string } | undefined;
+ async setRemoteDescription(description?: { sdp?: string }) {
this.remoteReady = true;
+ this.remote = description;
}
async setLocalDescription() {}
async createAnswer() {
@@ -94,18 +96,36 @@ function setup() {
reply,
};
disposers.push(startDesktopCaptureHost(api as any));
- const offer = (audio = true, overlay = true, nativeAudio = false) =>
+ const offer = (
+ audio = true,
+ overlay = true,
+ nativeAudio = false,
+ quality?: 'auto' | 'saver' | 'hd',
+ fps: 30 | 60 = 30,
+ sdp = 'offer',
+ ) =>
command({
id: 'offer',
op: 'offer',
lease: 'lease',
- sdp: 'offer',
+ sdp,
attemptId: 'attempt',
sourceId: 'screen:1',
nativeCapture: true,
nativeAudio,
cursorOverlay: overlay,
- settings: { audio, fps: 30 },
+ settings: { audio, fps, ...(quality ? { quality } : {}) },
+ });
+ const offerWithoutSettings = (sdp: string) =>
+ command({
+ id: 'offer',
+ op: 'offer',
+ lease: 'lease',
+ sdp,
+ attemptId: 'attempt',
+ sourceId: 'screen:1',
+ nativeCapture: true,
+ cursorOverlay: true,
});
return {
api,
@@ -115,6 +135,7 @@ function setup() {
capture,
reply,
offer,
+ offerWithoutSettings,
reset: (resume = false) =>
command({ op: 'capture-reset', lease: 'lease', nativeAudio: resume }),
stop: () => command({ op: 'stop' }),
@@ -137,16 +158,64 @@ function nativeSound() {
return { sound, close };
}
-it('allows congestion-driven resolution and frame-rate reduction without restarting capture', async () => {
+it.each([
+ [undefined, 60, 'maintain-framerate', 60, 20_000_000, ''],
+ ['auto', 60, 'maintain-framerate', 60, 20_000_000, ''],
+ ['saver', 60, 'maintain-framerate', 30, 2_000_000, ''],
+ ['hd', 60, 'maintain-resolution', 60, 20_000_000, 'text'],
+ ['hd', 30, 'maintain-resolution', 30, 20_000_000, 'text'],
+] as const)(
+ 'applies the %s tier at %i fps as sender ceilings without restarting capture',
+ async (quality, fps, degradationPreference, maxFramerate, maxBitrate, contentHint) => {
+ const h = setup();
+ h.offer(false, true, false, quality, fps);
+ await flush();
+ expect(h.peers[0].video.setParameters).toHaveBeenCalledExactlyOnceWith({
+ degradationPreference,
+ encodings: [{ maxFramerate, maxBitrate }],
+ });
+ expect(vi.mocked(nativeCaptureStream).mock.calls.at(-1)?.[4]).toBe(maxFramerate);
+ expect((h.video.video as { contentHint?: string }).contentHint).toBe(contentHint);
+ expect(h.peers).toHaveLength(1);
+ expect(h.reply).toHaveBeenCalledWith('offer', 'answer');
+ expect(h.nativeStop).not.toHaveBeenCalled();
+ },
+);
+
+it('keeps full resolution while the screen is still only for tiers that ask for it', async () => {
const h = setup();
- h.offer(false);
+ h.offer(false, true, false, 'auto');
await flush();
- expect(h.peers[0].video.setParameters).toHaveBeenCalledExactlyOnceWith({
- degradationPreference: 'balanced', encodings: [{ maxFramerate: 30 }],
- });
- expect(h.peers).toHaveLength(1);
- expect(h.reply).toHaveBeenCalledWith('offer', 'answer');
- expect(h.nativeStop).not.toHaveBeenCalled();
+ const onMotion = vi.mocked(nativeCaptureStream).mock.calls.at(-1)?.[5];
+ expect(onMotion).toBeTypeOf('function');
+ const video = h.peers[0].video.setParameters;
+ onMotion!(false);
+ await flush();
+ expect(video).toHaveBeenLastCalledWith(
+ expect.objectContaining({ degradationPreference: 'maintain-resolution' }),
+ );
+ onMotion!(true);
+ await flush();
+ expect(video).toHaveBeenLastCalledWith(
+ expect.objectContaining({ degradationPreference: 'maintain-framerate' }),
+ );
+
+ const hd = setup();
+ hd.offer(false, true, false, 'hd');
+ await flush();
+ expect(vi.mocked(nativeCaptureStream).mock.calls.at(-1)?.[5]).toBeUndefined();
+});
+
+it('passes the tier bandwidth floor to the viewer offer, leaving legacy offers untouched', async () => {
+ const sdp = 'v=0\r\nm=video 9 UDP/TLS/RTP/SAVPF 100\r\na=rtpmap:100 VP8/90000\r\n';
+ const h = setup();
+ h.offer(false, true, false, 'saver', 30, sdp);
+ await flush();
+ expect(h.peers[0].remote?.sdp).toContain('a=fmtp:100 x-google-start-bitrate=1500;');
+ const legacy = setup();
+ legacy.offerWithoutSettings(sdp);
+ await flush();
+ expect(legacy.peers.at(-1)?.remote?.sdp).toBe(sdp);
});
it('clears locked audio and restores its existing sender after unlock without Chromium capture', async () => {
diff --git a/apps/desktop/src/renderer/features/remote-desktop/__tests__/nativeCaptureStream.test.ts b/apps/desktop/src/renderer/features/remote-desktop/__tests__/nativeCaptureStream.test.ts
index ef14b9effa..e0b03a1b8c 100644
--- a/apps/desktop/src/renderer/features/remote-desktop/__tests__/nativeCaptureStream.test.ts
+++ b/apps/desktop/src/renderer/features/remote-desktop/__tests__/nativeCaptureStream.test.ts
@@ -85,3 +85,35 @@ it('clears the independent cursor when falling back to frames with a baked point
expect(cursor).toHaveBeenLastCalledWith(null);
owner.stop();
});
+it('reports still after a quiet second and moving again on a large change', async () => {
+ const h = setup();
+ let pixels = new Uint8ClampedArray(64 * 36 * 4);
+ h.canvas.getContext = () =>
+ ({
+ drawImage: h.drawImage,
+ clearRect: vi.fn(),
+ getImageData: () => ({ data: pixels }),
+ }) as never;
+ const onMotion = vi.fn();
+ const owner = await nativeCaptureStream(
+ async () => 'anBlZw==',
+ () => true,
+ vi.fn(),
+ undefined,
+ 30,
+ onMotion,
+ );
+ await vi.advanceTimersByTimeAsync(900);
+ expect(onMotion).not.toHaveBeenCalled();
+ await vi.advanceTimersByTimeAsync(300);
+ expect(onMotion).toHaveBeenLastCalledWith(false);
+ // A single pixel (caret, pulsing button) is not motion.
+ pixels = pixels.slice();
+ pixels.set([255, 255, 255, 255], 400);
+ await vi.advanceTimersByTimeAsync(200);
+ expect(onMotion).toHaveBeenCalledTimes(1);
+ pixels = new Uint8ClampedArray(64 * 36 * 4).fill(200);
+ await vi.advanceTimersByTimeAsync(100);
+ expect(onMotion).toHaveBeenLastCalledWith(true);
+ owner.stop();
+});
diff --git a/apps/desktop/src/renderer/features/remote-desktop/__tests__/viewerControl.test.ts b/apps/desktop/src/renderer/features/remote-desktop/__tests__/viewerControl.test.ts
index dec12189d5..f9445d91c4 100644
--- a/apps/desktop/src/renderer/features/remote-desktop/__tests__/viewerControl.test.ts
+++ b/apps/desktop/src/renderer/features/remote-desktop/__tests__/viewerControl.test.ts
@@ -332,13 +332,13 @@ it('coalesces rapid quality changes and waits for the current negotiation to pre
await fixture();
present();
runtime.receive.mockClear();
- controller.settings({ bitrate: 2000000 });
- controller.settings({ bitrate: 20000000 });
+ controller.settings({ quality: 'saver' });
+ controller.settings({ quality: 'hd' });
await vi.advanceTimersByTimeAsync(0);
expect(
runtime.receive.mock.calls.filter(([message]) => message.type === 'videoSettings'),
).toHaveLength(1);
- controller.settings({ bitrate: 8000000 });
+ controller.settings({ quality: 'auto' });
await vi.advanceTimersByTimeAsync(0);
expect(
runtime.receive.mock.calls.filter(([message]) => message.type === 'videoSettings'),
@@ -347,7 +347,7 @@ it('coalesces rapid quality changes and waits for the current negotiation to pre
expect(
runtime.receive.mock.calls.filter(([message]) => message.type === 'videoSettings'),
).toHaveLength(2);
- expect(snapshot.settings.bitrate).toBe(8000000);
+ expect(snapshot.settings.quality).toBe('auto');
});
it('expires stale bitrate and latency samples', async () => {
@@ -370,9 +370,9 @@ it('does not let a fallback screenshot interrupt a pending video-settings negoti
await fixture();
runtime.post?.({ type: 'framePresented', epoch: 'lease' });
runtime.receive.mockClear();
- controller.settings({ bitrate: 2000000 });
+ controller.settings({ quality: 'saver' });
await vi.advanceTimersByTimeAsync(0);
- controller.settings({ bitrate: 8000000 });
+ controller.settings({ quality: 'auto' });
await vi.advanceTimersByTimeAsync(0);
runtime.post?.({ type: 'framePresented', epoch: 'lease' });
expect(
diff --git a/apps/desktop/src/renderer/features/remote-desktop/__tests__/viewerWindow.test.tsx b/apps/desktop/src/renderer/features/remote-desktop/__tests__/viewerWindow.test.tsx
index 01986a8abc..d4a68519d4 100644
--- a/apps/desktop/src/renderer/features/remote-desktop/__tests__/viewerWindow.test.tsx
+++ b/apps/desktop/src/renderer/features/remote-desktop/__tests__/viewerWindow.test.tsx
@@ -86,7 +86,7 @@ it('confirms toolbar and native exits, keeps cancellation connected, and discard
displayId: '',
transport: 'direct',
latency: null,
- settings: { fps: 30, bitrate: 0, audio: true },
+ settings: { fps: 30, quality: 'auto', audio: true },
ready: true,
preferences: {
audio: true,
@@ -191,7 +191,7 @@ it('hides view-only controls and enables desktop actions only after control is c
displayId: 'one',
transport: 'direct',
latency: null,
- settings: { fps: 30, bitrate: 0, audio: false },
+ settings: { fps: 30, quality: 'auto', audio: false },
caps: {
version: 1,
enabled: true,
@@ -324,7 +324,7 @@ it.each([
displayId: 'one',
transport,
latency: null,
- settings: { fps: 30, bitrate: 0, audio: false },
+ settings: { fps: 30, quality: 'auto', audio: false },
}),
);
const toolbar = within(view.container.querySelector('header')!);
diff --git a/apps/desktop/src/renderer/features/remote-desktop/captureHost.ts b/apps/desktop/src/renderer/features/remote-desktop/captureHost.ts
index 2b21403358..e1c55a9e72 100644
--- a/apps/desktop/src/renderer/features/remote-desktop/captureHost.ts
+++ b/apps/desktop/src/renderer/features/remote-desktop/captureHost.ts
@@ -8,6 +8,11 @@ import {
type RemoteDesktopCursor,
} from '@cindy/device-link';
import { DESKTOP_AUDIO_RETRY_MS, type DesktopCaptureApi } from '../../../shared/remoteDesktop';
+import {
+ desktopVideoFramerate,
+ desktopVideoProfile,
+ withDesktopBitrateHints,
+} from '../../../shared/remoteDesktopQuality';
import { nativeCaptureStream } from './nativeCaptureStream';
import { PortalCaptureStream } from './portalCaptureStream';
import { nativeAudioStream } from './nativeAudioStream';
@@ -155,11 +160,33 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
void (async () => {
try {
let captureSettled: Promise<void> = Promise.resolve();
+ const profile = desktopVideoProfile(command.settings);
+ const fps = desktopVideoFramerate(command.settings);
+ // Native capture reports whether the screen is moving; tiers that stay
+ // sharp when still trade frame rate for resolution only while still.
+ let moving = true;
+ let videoSender: RTCRtpSender | null = null;
+ let preference = Promise.resolve();
+ const degradation = () => (moving ? profile.degradation : 'maintain-resolution');
+ const onMotion = (next: boolean) => {
+ moving = next;
+ preference = preference
+ .then(async () => {
+ const sender = videoSender;
+ if (!sender || current !== generation) return;
+ const parameters = sender.getParameters();
+ if (!parameters.encodings?.length) return;
+ if (parameters.degradationPreference === degradation()) return;
+ parameters.degradationPreference = degradation();
+ await sender.setParameters(parameters);
+ })
+ .catch(() => {});
+ };
const capture = () =>
navigator.mediaDevices.getDisplayMedia({
audio: command.settings?.audio === true,
video: {
- frameRate: { ideal: command.settings?.fps ?? 30, max: command.settings?.fps ?? 30 },
+ frameRate: { ideal: fps, max: fps },
},
});
const boundedCapture = async () => {
@@ -204,9 +231,8 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
(value) => {
if (current === generation) latestCursor = value;
},
- command.cursorOverlay || command.continuousNativeCapture
- ? (command.settings?.fps ?? 30)
- : 15,
+ command.cursorOverlay || command.continuousNativeCapture ? fps : 15,
+ command.settings && profile.sharpWhenStill ? onMotion : undefined,
);
if (current !== generation) {
result.stop();
@@ -293,7 +319,9 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
const replacement = await nativeStream();
const sender = rtc.getSenders().find((item) => item.track?.kind === 'video');
if (!sender || current !== generation) throw new Error('DESKTOP_VIDEO_STOPPED');
- await sender.replaceTrack(replacement.getVideoTracks()[0]);
+ const video = replacement.getVideoTracks()[0];
+ if (video) video.contentHint = profile.contentHint;
+ await sender.replaceTrack(video);
captured.getVideoTracks().forEach((track) => {
track.onended = null;
track.onmute = null;
@@ -391,8 +419,17 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
}
};
};
- stream.getTracks().forEach((track) => rtc.addTrack(track, captured));
- await rtc.setRemoteDescription({ type: 'offer', sdp: command.sdp });
+ stream.getTracks().forEach((track) => {
+ if (track.kind === 'video') track.contentHint = profile.contentHint;
+ rtc.addTrack(track, captured);
+ });
+ await rtc.setRemoteDescription({
+ type: 'offer',
+ sdp:
+ command.settings && command.sdp
+ ? withDesktopBitrateHints(command.sdp, profile)
+ : command.sdp,
+ });
// Negotiate audio now even when permission is not ready. replaceTrack
// can fill this sender later without interrupting video or the data channel.
const audioTransceiver =
@@ -440,12 +477,13 @@ export function startDesktopCaptureHost(api: DesktopCaptureApi): () => void {
if (sender.track?.kind !== 'video' || !command.settings) continue;
const parameters = sender.getParameters();
if (!parameters.encodings?.length) continue;
- // Let WebRTC's congestion controller trade resolution AND frame rate
- // below the user's ceilings without renegotiating the capture lease.
- parameters.degradationPreference = 'balanced';
+ // Below these ceilings WebRTC's congestion controller picks the rate;
+ // the tier decides whether resolution or frame rate gives way first.
+ parameters.degradationPreference = degradation();
+ videoSender = sender;
for (const encoding of parameters.encodings) {
- encoding.maxFramerate = command.settings.fps;
- if (command.settings.bitrate) encoding.maxBitrate = command.settings.bitrate;
+ encoding.maxFramerate = fps;
+ encoding.maxBitrate = profile.maxBitrate;
}
await sender.setParameters(parameters);
}
diff --git a/apps/desktop/src/renderer/features/remote-desktop/nativeCaptureStream.ts b/apps/desktop/src/renderer/features/remote-desktop/nativeCaptureStream.ts
index 3e79e75867..954813ad0f 100644
--- a/apps/desktop/src/renderer/features/remote-desktop/nativeCaptureStream.ts
+++ b/apps/desktop/src/renderer/features/remote-desktop/nativeCaptureStream.ts
@@ -1,4 +1,5 @@
import type { RemoteDesktopCursor, RemoteDesktopCursorFrame } from '@cindy/device-link';
+import { DESKTOP_MOTION, desktopFrameChange } from '../../../shared/remoteDesktopQuality';
/** Converts the bounded native fallback frames into the existing WebRTC video
* transport. Serial pulls/decode keep both IPC and image memory bounded.
*/
@@ -8,10 +9,34 @@ export async function nativeCaptureStream(
failed: () => void,
cursor: (value: RemoteDesktopCursor | null) => void = () => {},
fps = 15,
+ onMotion?: (moving: boolean) => void,
): Promise<{ stream: MediaStream; stop(): void; clear(): void }> {
const canvas = document.createElement('canvas');
const context = canvas.getContext('2d');
if (!context) throw new Error('DESKTOP_VIDEO_UNAVAILABLE');
+ // A tiny thumbnail separates large changes (scrolling, video) from small
+ // localized activity (caret, pulsing button) at negligible cost.
+ const thumb = onMotion ? document.createElement('canvas') : null;
+ const thumbContext = thumb?.getContext('2d', { willReadFrequently: true });
+ if (thumb) thumb.width = 64;
+ if (thumb) thumb.height = 36;
+ let lastThumb: Uint8ClampedArray | null = null;
+ let lastMotion = 0;
+ let moving = true;
+ const observe = (bitmap: ImageBitmap) => {
+ if (!thumbContext || !onMotion) return;
+ thumbContext.drawImage(bitmap, 0, 0, 64, 36);
+ const next = thumbContext.getImageData(0, 0, 64, 36).data;
+ const now = performance.now();
+ if (!lastThumb || desktopFrameChange(lastThumb, next) > DESKTOP_MOTION.changedShare)
+ lastMotion = now;
+ lastThumb = next;
+ const nextMoving = now - lastMotion < DESKTOP_MOTION.stillAfterMs;
+ if (nextMoving !== moving) {
+ moving = nextMoving;
+ onMotion(moving);
+ }
+ };
let stopped = false;
let timer: ReturnType<typeof setTimeout> | null = null;
let epoch = 0;
@@ -33,6 +58,7 @@ export async function nativeCaptureStream(
canvas.height = bitmap.height;
}
context.drawImage(bitmap, 0, 0);
+ observe(bitmap);
// A fallback frame includes the pointer in its pixels. Clear the last
// independent cursor so switching backends cannot leave two pointers.
cursor(typeof frame === 'string' ? null : frame.cursor);
diff --git a/apps/desktop/src/renderer/features/remote-desktop/viewerController.ts b/apps/desktop/src/renderer/features/remote-desktop/viewerController.ts
index 3e7ab162a6..513ffe2f08 100644
--- a/apps/desktop/src/renderer/features/remote-desktop/viewerController.ts
+++ b/apps/desktop/src/renderer/features/remote-desktop/viewerController.ts
@@ -72,7 +72,7 @@ export class DesktopViewerController {
displayId: '',
transport: '',
latency: null,
- settings: { fps: 30, bitrate: 0, audio: true },
+ settings: { fps: 30, quality: 'auto', audio: true },
ready: false,
preferences: { ...DEFAULT_VIEWER_PREFERENCES },
safety: { privacyActive: false, notice: null, clipboardProgress: null },
diff --git a/apps/desktop/src/renderer/i18n/locales/en/common.json b/apps/desktop/src/renderer/i18n/locales/en/common.json
index d920b48191..4eb8fceeb2 100644
--- a/apps/desktop/src/renderer/i18n/locales/en/common.json
+++ b/apps/desktop/src/renderer/i18n/locales/en/common.json
@@ -305,9 +305,9 @@
"fps": "Frame rate",
"quality": "Video quality",
"automatic": "Automatic",
- "smooth": "Smooth",
- "balanced": "Balanced",
- "clear": "Clear",
+ "saver": "Data saver",
+ "hd": "HD",
+ "qualityHint": "Auto keeps motion smooth and sharpens when your network allows. Data saver limits usage (up to 30 fps). HD keeps text sharp and lowers the frame rate when bandwidth is short.",
"resolution": "Computer resolution",
"clipboardFailed": "Could not transfer clipboard content. Check control access and try again.",
"settingsFailed": "Could not save this setting. Try again.",
diff --git a/apps/desktop/src/renderer/i18n/locales/ja/common.json b/apps/desktop/src/renderer/i18n/locales/ja/common.json
index 4e68e3913a..17615e6fd9 100644
--- a/apps/desktop/src/renderer/i18n/locales/ja/common.json
+++ b/apps/desktop/src/renderer/i18n/locales/ja/common.json
@@ -305,9 +305,9 @@
"fps": "フレームレート",
"quality": "画質",
"automatic": "自動",
- "smooth": "スムーズ",
- "balanced": "バランス",
- "clear": "高画質",
+ "saver": "データ節約",
+ "hd": "高画質",
+ "qualityHint": "自動は滑らかさを優先し、通信に余裕があれば鮮明にします。データ節約は通信量を抑えます(最大 30 fps)。高画質は文字の鮮明さを優先し、帯域が足りないときはフレームレートを下げます。",
"resolution": "コンピュータの解像度",
"clipboardFailed": "クリップボードの内容を転送できませんでした。操作権限を確認して再試行してください。",
"settingsFailed": "設定を保存できませんでした。再試行してください。",
diff --git a/apps/desktop/src/renderer/i18n/locales/ko/common.json b/apps/desktop/src/renderer/i18n/locales/ko/common.json
index b27687ef91..e475e71bbb 100644
--- a/apps/desktop/src/renderer/i18n/locales/ko/common.json
+++ b/apps/desktop/src/renderer/i18n/locales/ko/common.json
@@ -305,9 +305,9 @@
"fps": "프레임 속도",
"quality": "화질",
"automatic": "자동",
- "smooth": "원활",
- "balanced": "균형",
- "clear": "선명",
+ "saver": "데이터 절약",
+ "hd": "고화질",
+ "qualityHint": "자동은 부드러움을 우선하고 네트워크가 좋으면 더 선명하게 표시합니다. 데이터 절약은 사용량을 줄입니다(최대 30fps). 고화질은 글자 선명도를 우선하며 대역폭이 부족하면 프레임 속도를 낮춥니다.",
"resolution": "컴퓨터 해상도",
"clipboardFailed": "클립보드 내용을 전송하지 못했습니다. 제어 권한을 확인하고 다시 시도하세요.",
"settingsFailed": "설정을 저장하지 못했습니다. 다시 시도하세요.",
diff --git a/apps/desktop/src/renderer/i18n/locales/zh-CN/common.json b/apps/desktop/src/renderer/i18n/locales/zh-CN/common.json
index 82c4baa679..a68570ce9d 100644
--- a/apps/desktop/src/renderer/i18n/locales/zh-CN/common.json
+++ b/apps/desktop/src/renderer/i18n/locales/zh-CN/common.json
@@ -305,9 +305,9 @@
"fps": "帧率",
"quality": "画质",
"automatic": "自动",
- "smooth": "流畅",
- "balanced": "均衡",
- "clear": "清晰",
+ "saver": "省流",
+ "hd": "高清",
+ "qualityHint": "自动优先保证流畅,网络好时尽量清晰;省流会限制流量(最高 30 帧);高清优先保证文字清晰,网络不够时会降低帧率。",
"resolution": "电脑分辨率",
"clipboardFailed": "剪贴板内容传输失败,请检查控制权限后重试。",
"settingsFailed": "设置未能保存,请重试。",
diff --git a/apps/desktop/src/renderer/i18n/locales/zh-TW/common.json b/apps/desktop/src/renderer/i18n/locales/zh-TW/common.json
index dd988ccf6a..a7a2d76946 100644
--- a/apps/desktop/src/renderer/i18n/locales/zh-TW/common.json
+++ b/apps/desktop/src/renderer/i18n/locales/zh-TW/common.json
@@ -305,9 +305,9 @@
"fps": "影格率",
"quality": "畫質",
"automatic": "自動",
- "smooth": "流暢",
- "balanced": "均衡",
- "clear": "清晰",
+ "saver": "省流量",
+ "hd": "高清",
+ "qualityHint": "自動優先確保流暢,網路良好時盡量清晰;省流量會限制用量(最高 30 幀);高清優先確保文字清晰,頻寬不足時會降低幀率。",
"resolution": "電腦解析度",
"clipboardFailed": "剪貼簿內容傳輸失敗,請檢查控制權限後重試。",
"settingsFailed": "設定未能儲存,請重試。",
diff --git a/apps/desktop/src/shared/__tests__/remoteDesktopQuality.test.ts b/apps/desktop/src/shared/__tests__/remoteDesktopQuality.test.ts
new file mode 100644
index 0000000000..adac1dd3a2
--- /dev/null
+++ b/apps/desktop/src/shared/__tests__/remoteDesktopQuality.test.ts
@@ -0,0 +1,86 @@
+import { describe, expect, it } from 'vitest';
+import {
+ DESKTOP_MOTION,
+ desktopFrameChange,
+ desktopVideoFramerate,
+ desktopVideoProfile,
+ withDesktopBitrateHints,
+} from '../remoteDesktopQuality';
+
+const offer = [
+ 'v=0',
+ 'm=audio 9 UDP/TLS/RTP/SAVPF 111',
+ 'a=rtpmap:111 opus/48000/2',
+ 'a=fmtp:111 minptime=10;useinbandfec=1',
+ 'm=video 9 UDP/TLS/RTP/SAVPF 96 97 100 101',
+ 'a=rtpmap:96 H264/90000',
+ 'a=fmtp:96 packetization-mode=1;x-google-min-bitrate=30',
+ 'a=rtpmap:97 rtx/90000',
+ 'a=fmtp:97 apt=96',
+ 'a=rtpmap:100 VP8/90000',
+ 'a=rtpmap:101 rtx/90000',
+ 'a=fmtp:101 apt=100',
+ 'm=application 9 UDP/DTLS/SCTP webrtc-datachannel',
+ '',
+].join('\r\n');
+
+describe('remote desktop quality tiers', () => {
+ it('lets the host own each tier, including the frame-rate ceiling', () => {
+ expect(desktopVideoProfile().degradation).toBe('maintain-framerate');
+ expect(desktopVideoProfile({ fps: 60, quality: 'hd', audio: false })).toMatchObject({
+ degradation: 'maintain-resolution',
+ sharpWhenStill: false,
+ });
+ expect(desktopVideoFramerate({ fps: 60, quality: 'saver', audio: false })).toBe(30);
+ expect(desktopVideoFramerate({ fps: 60, quality: 'auto', audio: false })).toBe(60);
+ for (const quality of ['auto', 'saver', 'hd'] as const) {
+ const p = desktopVideoProfile({ fps: 30, quality, audio: false });
+ expect(p.minBitrateKbps).toBeLessThanOrEqual(p.startBitrateKbps);
+ expect(p.startBitrateKbps * 1000).toBeLessThanOrEqual(p.maxBitrate);
+ }
+ });
+
+ it('adds bandwidth hints to every video media codec only', () => {
+ const profile = desktopVideoProfile({ fps: 30, quality: 'saver', audio: false });
+ const lines = withDesktopBitrateHints(offer, profile).split('\r\n');
+ const hints = 'x-google-start-bitrate=1500;x-google-min-bitrate=500;x-google-max-bitrate=2000';
+ expect(lines).toContain(`a=fmtp:96 packetization-mode=1;${hints}`);
+ expect(lines).toContain(`a=fmtp:100 ${hints}`);
+ expect(lines).toContain('a=fmtp:97 apt=96');
+ expect(lines).toContain('a=fmtp:101 apt=100');
+ expect(lines).toContain('a=fmtp:111 minptime=10;useinbandfec=1');
+ // The inserted VP8 line stays inside the video section, before the next m= line.
+ expect(lines.indexOf(`a=fmtp:100 ${hints}`)).toBeLessThan(
+ lines.indexOf('m=application 9 UDP/DTLS/SCTP webrtc-datachannel'),
+ );
+ expect(lines.at(-1)).toBe('');
+ });
+
+ it('rewrites an fmtp line that precedes its rtpmap instead of adding a second one', () => {
+ const profile = desktopVideoProfile();
+ const reordered = [
+ 'm=video 9 UDP/TLS/RTP/SAVPF 100',
+ 'a=fmtp:100 x-google-min-bitrate=30',
+ 'a=rtpmap:100 VP8/90000',
+ '',
+ ].join('\n');
+ const lines = withDesktopBitrateHints(reordered, profile).split('\n');
+ expect(lines.filter((line) => line.startsWith('a=fmtp:100'))).toEqual([
+ 'a=fmtp:100 x-google-start-bitrate=4000;x-google-min-bitrate=1000;x-google-max-bitrate=20000',
+ ]);
+ expect(lines.at(-1)).toBe('');
+ });
+
+ it('separates large screen changes from small localized activity', () => {
+ const frame = (fill: number) => new Uint8ClampedArray(64 * 36 * 4).fill(fill);
+ const still = frame(100);
+ expect(desktopFrameChange(still, frame(100))).toBe(0);
+ const caret = frame(100);
+ caret.set([255, 255, 255, 255], 4 * 500);
+ expect(desktopFrameChange(still, caret)).toBeLessThan(DESKTOP_MOTION.changedShare);
+ const scrolled = frame(100);
+ scrolled.fill(140, 0, scrolled.length / 4);
+ expect(desktopFrameChange(still, scrolled)).toBeGreaterThan(DESKTOP_MOTION.changedShare);
+ expect(desktopFrameChange(still, new Uint8ClampedArray(4))).toBe(1);
+ });
+});
diff --git a/apps/desktop/src/shared/remoteDesktopQuality.ts b/apps/desktop/src/shared/remoteDesktopQuality.ts
new file mode 100644
index 0000000000..ee1242b1bb
--- /dev/null
+++ b/apps/desktop/src/shared/remoteDesktopQuality.ts
@@ -0,0 +1,148 @@
+import type { RemoteDesktopVideoSettings } from '@cindy/device-link';
+
+/** Host-owned meaning of each viewer quality tier. Viewers only send the tier,
+ * so these values can be tuned without a coordinated viewer release.
+ */
+export interface DesktopVideoProfile {
+ /** WebRTC sender ceiling; congestion control still picks the rate below it. */
+ maxBitrate: number;
+ /** Bandwidth-estimate floor and starting point, in kbps (SDP x-google-* hints).
+ * A near-static desktop sends almost nothing, so one jitter spike can otherwise
+ * collapse the estimate to ~100 kbps and pin the encoder to a low resolution. */
+ minBitrateKbps: number;
+ startBitrateKbps: number;
+ /** Upper bound on the viewer's requested frame rate. */
+ maxFramerate: 30 | 60;
+ /** What WebRTC gives up first while the screen is changing. */
+ degradation: 'maintain-framerate' | 'maintain-resolution';
+ /** While the screen is (nearly) still, keep full resolution regardless of tier:
+ * WebRTC otherwise chooses resolution from bitrate alone, even for static frames. */
+ sharpWhenStill: boolean;
+ /** Track hint; `text` makes the encoder favour sharp screen content. */
+ contentHint: '' | 'text';
+ /** First JPEG quality tried for native capture frames (0–1). */
+ jpegQuality: number;
+ /** macOS: capture physical pixels with this long-edge cap; 0 keeps logical points. */
+ physicalMaxEdge: number;
+ /** macOS: largest native JPEG frame before quality/scale fallback. */
+ maxFrameBytes: number;
+ /** Windows helper still reads the legacy bitrate to pick its JPEG budget. */
+ windowsBitrate: 0 | 8_000_000 | 20_000_000;
+}
+
+const PROFILES: Record<RemoteDesktopVideoSettings['quality'], DesktopVideoProfile> = {
+ // Smooth while moving; as sharp as HD whenever the screen is still.
+ auto: {
+ maxBitrate: 20_000_000,
+ minBitrateKbps: 1_000,
+ startBitrateKbps: 4_000,
+ maxFramerate: 60,
+ degradation: 'maintain-framerate',
+ sharpWhenStill: true,
+ contentHint: '',
+ jpegQuality: 0.8,
+ physicalMaxEdge: 2560,
+ maxFrameBytes: 1_500_000,
+ windowsBitrate: 8_000_000,
+ },
+ // Bounded traffic while moving, but still frames stay readable.
+ saver: {
+ maxBitrate: 2_000_000,
+ minBitrateKbps: 500,
+ startBitrateKbps: 1_500,
+ maxFramerate: 30,
+ degradation: 'maintain-framerate',
+ sharpWhenStill: true,
+ contentHint: '',
+ jpegQuality: 0.65,
+ physicalMaxEdge: 0,
+ maxFrameBytes: 1_000_000,
+ windowsBitrate: 0,
+ },
+ // Sharp text first; frame rate drops when the link cannot keep up.
+ hd: {
+ maxBitrate: 20_000_000,
+ minBitrateKbps: 1_000,
+ startBitrateKbps: 4_000,
+ maxFramerate: 60,
+ degradation: 'maintain-resolution',
+ sharpWhenStill: false,
+ contentHint: 'text',
+ jpegQuality: 0.9,
+ physicalMaxEdge: 3840,
+ maxFrameBytes: 3_000_000,
+ windowsBitrate: 20_000_000,
+ },
+};
+
+export function desktopVideoProfile(settings?: RemoteDesktopVideoSettings): DesktopVideoProfile {
+ return PROFILES[settings?.quality ?? 'auto'];
+}
+
+/** Effective capture/encode frame rate for the requested settings. */
+export function desktopVideoFramerate(settings?: RemoteDesktopVideoSettings): 30 | 60 {
+ return Math.min(settings?.fps ?? 30, desktopVideoProfile(settings).maxFramerate) as 30 | 60;
+}
+
+const BITRATE_HINTS = ['x-google-start-bitrate', 'x-google-min-bitrate', 'x-google-max-bitrate'];
+const MEDIA_CODECS = /^(H264|VP8|VP9|AV1|H265)$/i;
+
+/** Adds bandwidth-estimate hints to the viewer's video codecs before the host
+ * applies the offer. libwebrtc reads them from the negotiated send codec; viewers
+ * that ignore x-google-* parameters are unaffected. */
+export function withDesktopBitrateHints(sdp: string, profile: DesktopVideoProfile): string {
+ const hints =
+ `x-google-start-bitrate=${profile.startBitrateKbps};` +
+ `x-google-min-bitrate=${profile.minBitrateKbps};` +
+ `x-google-max-bitrate=${Math.round(profile.maxBitrate / 1000)}`;
+ const eol = sdp.includes('\r\n') ? '\r\n' : '\n';
+ const lines = sdp.split(eol);
+ const trailing = lines.at(-1) === '' ? lines.pop() : undefined;
+ // Attribute order inside a media section is free, so collect the section's
+ // media payload types before rewriting any fmtp line.
+ const sections: string[][] = [[]];
+ for (const line of lines) {
+ if (line.startsWith('m=')) sections.push([]);
+ sections[sections.length - 1].push(line);
+ }
+ const out = sections.flatMap((section) => {
+ if (!section[0]?.startsWith('m=video')) return section;
+ const media = new Set<string>();
+ for (const line of section) {
+ const rtpmap = /^a=rtpmap:(\d+) ([^/]+)\//.exec(line);
+ if (rtpmap && MEDIA_CODECS.test(rtpmap[2])) media.add(rtpmap[1]);
+ }
+ const missing = new Set(media);
+ const rewritten = section.map((line) => {
+ const fmtp = /^a=fmtp:(\d+) (.*)$/.exec(line);
+ if (!fmtp || !media.has(fmtp[1])) return line;
+ missing.delete(fmtp[1]);
+ const kept = fmtp[2]
+ .split(';')
+ .filter((p) => p && !BITRATE_HINTS.includes(p.split('=')[0].trim()));
+ return `a=fmtp:${fmtp[1]} ${[...kept, hints].join(';')}`;
+ });
+ return [...rewritten, ...[...missing].map((pt) => `a=fmtp:${pt} ${hints}`)];
+ });
+ if (trailing !== undefined) out.push(trailing);
+ return out.join(eol);
+}
+
+/** Share of a frame that changed (0–1), from two equally sized RGBA thumbnails.
+ * Thumbnails average away small localized activity such as a blinking caret. */
+export function desktopFrameChange(previous: Uint8ClampedArray, next: Uint8ClampedArray): number {
+ if (previous.length !== next.length || !next.length) return 1;
+ let changed = 0;
+ for (let i = 0; i < next.length; i += 4) {
+ if (
+ Math.abs(previous[i] - next[i]) > 10 ||
+ Math.abs(previous[i + 1] - next[i + 1]) > 10 ||
+ Math.abs(previous[i + 2] - next[i + 2]) > 10
+ )
+ changed++;
+ }
+ return changed / (next.length / 4);
+}
+
+/** Moving only when a meaningful area changed; still again after a quiet second. */
+export const DESKTOP_MOTION = { changedShare: 0.02, stillAfterMs: 1_000 } as const;
diff --git a/apps/mobile/src/i18n/locales/en/remoteDesktop.json b/apps/mobile/src/i18n/locales/en/remoteDesktop.json
index 9b9802ca18..01213dca5f 100644
--- a/apps/mobile/src/i18n/locales/en/remoteDesktop.json
+++ b/apps/mobile/src/i18n/locales/en/remoteDesktop.json
@@ -124,10 +124,9 @@
"fps": "{{count}} fps",
"quality": "Quality",
"automatic": "Auto",
- "clear": "Clear",
- "highDefinition": "HD",
- "original": "Original",
- "qualityHint": "Auto adjusts picture quality to your connection. Higher quality needs more bandwidth; smoothness depends on your computer and network.",
+ "saver": "Data saver",
+ "hd": "HD",
+ "qualityHint": "Auto keeps motion smooth and sharpens when your network allows. Data saver limits usage (up to 30 fps). HD keeps text sharp and lowers the frame rate when bandwidth is short.",
"resolution": "Computer resolution",
"settingUnsupported": "This computer does not support this setting yet",
"loadingSettings": "Loading…",
diff --git a/apps/mobile/src/i18n/locales/ja/remoteDesktop.json b/apps/mobile/src/i18n/locales/ja/remoteDesktop.json
index a03b4fe518..5505e8ea31 100644
--- a/apps/mobile/src/i18n/locales/ja/remoteDesktop.json
+++ b/apps/mobile/src/i18n/locales/ja/remoteDesktop.json
@@ -124,10 +124,9 @@
"fps": "{{count}} fps",
"quality": "画質",
"automatic": "自動",
- "clear": "標準",
- "highDefinition": "高画質",
- "original": "オリジナル",
- "qualityHint": "自動では通信状況に合わせて画質を調整します。高画質ほど通信量が増えます。映像の滑らかさはパソコンと通信状況によって変わります。",
+ "saver": "データ節約",
+ "hd": "高画質",
+ "qualityHint": "自動は滑らかさを優先し、通信に余裕があれば鮮明にします。データ節約は通信量を抑えます(最大 30 fps)。高画質は文字の鮮明さを優先し、帯域が足りないときはフレームレートを下げます。",
"resolution": "パソコンの解像度",
"settingUnsupported": "このパソコンはまだこの設定に対応していません",
"loadingSettings": "読み込み中…",
diff --git a/apps/mobile/src/i18n/locales/ko/remoteDesktop.json b/apps/mobile/src/i18n/locales/ko/remoteDesktop.json
index cc36dcc2b9..0371f12b85 100644
--- a/apps/mobile/src/i18n/locales/ko/remoteDesktop.json
+++ b/apps/mobile/src/i18n/locales/ko/remoteDesktop.json
@@ -124,10 +124,9 @@
"fps": "{{count}} fps",
"quality": "화질",
"automatic": "자동",
- "clear": "표준",
- "highDefinition": "고화질",
- "original": "원본",
- "qualityHint": "자동은 네트워크 상태에 맞춰 화질을 조절합니다. 화질이 높을수록 더 많은 대역폭이 필요하며, 영상의 부드러움은 컴퓨터와 네트워크 상태에 따라 달라집니다.",
+ "saver": "데이터 절약",
+ "hd": "고화질",
+ "qualityHint": "자동은 부드러움을 우선하고 네트워크가 좋으면 더 선명하게 표시합니다. 데이터 절약은 사용량을 줄입니다(최대 30fps). 고화질은 글자 선명도를 우선하며 대역폭이 부족하면 프레임 속도를 낮춥니다.",
"resolution": "컴퓨터 해상도",
"settingUnsupported": "이 컴퓨터는 아직 이 설정을 지원하지 않습니다",
"loadingSettings": "불러오는 중…",
diff --git a/apps/mobile/src/i18n/locales/zh-CN/remoteDesktop.json b/apps/mobile/src/i18n/locales/zh-CN/remoteDesktop.json
index a712156257..be49728471 100644
--- a/apps/mobile/src/i18n/locales/zh-CN/remoteDesktop.json
+++ b/apps/mobile/src/i18n/locales/zh-CN/remoteDesktop.json
@@ -124,10 +124,9 @@
"fps": "{{count}} 帧",
"quality": "画质",
"automatic": "自动",
- "clear": "清晰",
- "highDefinition": "高清",
- "original": "原画",
- "qualityHint": "自动会根据网络调整画质。画质越高,需要的网络带宽越大;实际流畅度取决于电脑和网络。",
+ "saver": "省流",
+ "hd": "高清",
+ "qualityHint": "自动优先保证流畅,网络好时尽量清晰;省流会限制流量(最高 30 帧);高清优先保证文字清晰,网络不够时会降低帧率。",
"resolution": "电脑分辨率",
"settingUnsupported": "当前电脑端暂不支持此设置",
"loadingSettings": "读取中…",
diff --git a/apps/mobile/src/i18n/locales/zh-TW/remoteDesktop.json b/apps/mobile/src/i18n/locales/zh-TW/remoteDesktop.json
index e7c09ecbb0..80c4b4f814 100644
--- a/apps/mobile/src/i18n/locales/zh-TW/remoteDesktop.json
+++ b/apps/mobile/src/i18n/locales/zh-TW/remoteDesktop.json
@@ -124,10 +124,9 @@
"fps": "{{count}} 幀",
"quality": "畫質",
"automatic": "自動",
- "clear": "清晰",
- "highDefinition": "高清",
- "original": "原畫",
- "qualityHint": "自動會依網路調整畫質。畫質越高,需要的網路頻寬越大;實際流暢度取決於電腦和網路。",
+ "saver": "省流量",
+ "hd": "高清",
+ "qualityHint": "自動優先確保流暢,網路良好時盡量清晰;省流量會限制用量(最高 30 幀);高清優先確保文字清晰,頻寬不足時會降低幀率。",
"resolution": "電腦解析度",
"settingUnsupported": "目前電腦端尚不支援此設定",
"loadingSettings": "讀取中…",
diff --git a/apps/mobile/src/remote-desktop/RemoteDesktopDisplaySettings.tsx b/apps/mobile/src/remote-desktop/RemoteDesktopDisplaySettings.tsx
index 47769d7b08..8e4be2037c 100644
--- a/apps/mobile/src/remote-desktop/RemoteDesktopDisplaySettings.tsx
+++ b/apps/mobile/src/remote-desktop/RemoteDesktopDisplaySettings.tsx
@@ -9,9 +9,11 @@ import {
import SegmentedControl from "@expo/ui/community/segmented-control";
import { Check, ChevronDown, Maximize, RotateCcw } from "lucide-react-native";
import { useTranslation } from "react-i18next";
-import type {
- RemoteDesktopDisplayMode,
- RemoteDesktopVideoSettings,
+import {
+ REMOTE_DESKTOP_VIDEO_QUALITIES,
+ type RemoteDesktopDisplayMode,
+ type RemoteDesktopVideoQuality,
+ type RemoteDesktopVideoSettings,
} from "@cindy/device-link";
import { Text } from "@/components/AppText";
import { mobileInteractionStyles } from "@/components/mobileInteractionStyles";
@@ -30,6 +32,12 @@ import {
useTheme,
} from "@/theme";
+const QUALITY_LABELS = {
+ auto: "remoteDesktop.automatic",
+ saver: "remoteDesktop.saver",
+ hd: "remoteDesktop.hd",
+} as const satisfies Record<RemoteDesktopVideoQuality, string>;
+
type Props = {
video: {
supported: boolean;
@@ -161,7 +169,6 @@ export function RemoteDesktopDisplaySettings({
</View>
);
const fpsValues = [30, 60] as const;
- const qualityValues = [0, 2000000, 8000000, 20000000] as const;
const current = modes.find((mode) => mode.current);
const modeLabel = (mode: RemoteDesktopDisplayMode) =>
`${mode.width} × ${mode.height}${mode.native === true ? ` · ${t("remoteDesktop.nativeResolution")}` : ""}`;
@@ -246,11 +253,12 @@ export function RemoteDesktopDisplaySettings({
</Text>
{segmented(
"quality",
- ["automatic", "clear", "highDefinition", "original"].map((key) =>
- t(`remoteDesktop.${key}`),
+ REMOTE_DESKTOP_VIDEO_QUALITIES.map((quality) =>
+ t(QUALITY_LABELS[quality]),
),
- qualityValues.indexOf(video.settings.bitrate),
- (index) => video.onChange({ bitrate: qualityValues[index] }),
+ REMOTE_DESKTOP_VIDEO_QUALITIES.indexOf(video.settings.quality),
+ (index) =>
+ video.onChange({ quality: REMOTE_DESKTOP_VIDEO_QUALITIES[index] }),
)}
<Text style={hint}>{t("remoteDesktop.qualityHint")}</Text>
</View>
diff --git a/apps/mobile/src/remote-desktop/__tests__/screen.test.tsx b/apps/mobile/src/remote-desktop/__tests__/screen.test.tsx
index 64fa046fc1..634f306ed9 100644
--- a/apps/mobile/src/remote-desktop/__tests__/screen.test.tsx
+++ b/apps/mobile/src/remote-desktop/__tests__/screen.test.tsx
@@ -1625,7 +1625,7 @@ describe("remote desktop controls", () => {
supported: true,
busy,
modesSupported: false,
- settings: { fps: 30, bitrate: 0, audio: false },
+ settings: { fps: 30, quality: "auto", audio: false },
onChange,
readModes: async () => [],
onResolution: async () => {},
@@ -1672,7 +1672,7 @@ describe("remote desktop controls", () => {
modesSupported: false,
viewerDisplaySupported: supported,
onFitDisplay,
- settings: { fps: 30, bitrate: 0, audio: false },
+ settings: { fps: 30, quality: "auto", audio: false },
onChange: vi.fn(),
readModes: async () => [],
onResolution: async () => {},
@@ -2014,8 +2014,7 @@ describe("remote desktop controls", () => {
expect(changes()).toHaveLength(1);
await select("quality", 1);
await select("quality", 2);
- await select("quality", 3);
- expect(button("original").getAttribute("aria-selected")).toBe("true");
+ expect(button("hd").getAttribute("aria-selected")).toBe("true");
expect(changes()).toHaveLength(1);
expect(fixture.playback).not.toHaveBeenCalled();
await message({ type: terminal });
@@ -2025,7 +2024,7 @@ describe("remote desktop controls", () => {
requests()
.filter((r) => r.op === "offer")
.at(-1).settings,
- ).toMatchObject({ fps: 60, bitrate: 20000000 });
+ ).toMatchObject({ fps: 60, quality: "hd", bitrate: 20000000 });
await message({ type: "streaming" });
expect(changes()).toHaveLength(2);
await select("quality", 1);
@@ -2059,7 +2058,7 @@ describe("remote desktop controls", () => {
await message({ type: "streaming" });
act(() => button("operations").click());
act(() => button("displaySettings").click());
- await act(async () => button("original").click());
+ await act(async () => button("hd").click());
await message({ type: "fallback" });
expect(sent().filter((m) => m.type === "videoSettings")).toHaveLength(0);
await act(async () => finish({ sdp: "answer" }));
@@ -2076,7 +2075,7 @@ describe("remote desktop controls", () => {
'[data-testid="remoteDesktop.frameRateControl"] button',
)[1]
.click();
- button("original").click();
+ button("hd").click();
});
expect(
host
@@ -2085,7 +2084,7 @@ describe("remote desktop controls", () => {
)[1]
.getAttribute("aria-selected"),
).toBe("true");
- expect(button("original").getAttribute("aria-selected")).toBe("true");
+ expect(button("hd").getAttribute("aria-selected")).toBe("true");
});
it("cancels the PiP timeout when queued quality changes exit PiP", async () => {
fixture.systemAudio = true;
@@ -2110,7 +2109,7 @@ describe("remote desktop controls", () => {
act(() => button("operations").click());
await act(async () => button("smallWindow").click());
act(() => button("displaySettings").click());
- await act(async () => button("original").click());
+ await act(async () => button("hd").click());
await act(async () => finish({}));
expect(sent().filter((m) => m.type === "videoSettings")).toHaveLength(1);
await message({ type: "streaming" });
diff --git a/apps/mobile/src/remote-desktop/useVideoSettingsPreference.ts b/apps/mobile/src/remote-desktop/useVideoSettingsPreference.ts
index 8a9119551d..fa8e3a4f3d 100644
--- a/apps/mobile/src/remote-desktop/useVideoSettingsPreference.ts
+++ b/apps/mobile/src/remote-desktop/useVideoSettingsPreference.ts
@@ -8,7 +8,7 @@ let writes: Promise<void> = Promise.resolve();
export function useVideoSettingsPreference() {
const [settings, setSettings] = useState<RemoteDesktopVideoSettings>({
- fps: 30, bitrate: 0, audio: true,
+ fps: 30, quality: "auto", audio: true,
});
const [loaded, setLoaded] = useState(false);
const current = useRef(settings);
diff --git a/docs/dev-rules/protocol-compatibility.md b/docs/dev-rules/protocol-compatibility.md
index 11cecac9e3..97ce77589f 100644
--- a/docs/dev-rules/protocol-compatibility.md
+++ b/docs/dev-rules/protocol-compatibility.md
@@ -126,6 +126,24 @@ Mobile 未新增卡片入口。服务端无需改动。
`omarchyMenu`;缺省保留旧工具栏,不向旧主机发送新动作。旧端的 `desktop` 语义不变。
工作区切换作用于采集屏幕,菜单使用本机固定入口,所有操作沿用控制 lease 与撤权检查。
+## 远程桌面画质档位
+
+`offer.settings` 的画质由码率改为档位 `quality: "auto" | "saver" | "hd"`(自动/省流/高清)。
+控制端只表达意图,具体的码率上限、降级取舍(`auto`/`saver` 先降分辨率保帧数,`hd` 锁分辨率
+降帧数)、截屏分辨率与 JPEG 预算由被控端 `apps/desktop/src/shared/remoteDesktopQuality.ts`
+决定,调整数值无需两端同时发版。
+
+新控制端经 `remoteDesktopVideoSettingsWire` 同时发送档位与旧 `bitrate`(auto→0、saver→2M、
+hd→20M):旧被控端只校验 `bitrate` 并忽略 `quality`,无需新增能力声明。新被控端优先读取
+已知档位;档位缺失或不认识时按旧 `bitrate` 换算(0→auto、2M→saver、8M/20M→hd),因此旧
+控制端与未来新增档位都能降级连接。两者都无效时仍返回 `INVALID_REQUEST`。此变更不改 relay、
+不新增 channel,服务端无需改动。
+
+被控端在应用控制端 offer 前,仅为带 `settings` 的请求给视频编解码追加 `x-google-start-bitrate` /
+`x-google-min-bitrate` / `x-google-max-bitrate`,避免近静止画面因发送量过低导致带宽估计塌到
+百 kbps 级、分辨率被锁在低档。这些是 libwebrtc 对发送端生效的本地提示,不改变协商出的编解码;
+不识别它们的控制端不受影响,旧控制端(无 `settings`)的 offer 原样使用。
+
## 手机首页会话活动快照
### 可见历史优先读取
diff --git a/packages/device-link/src/__tests__/remoteDesktop.test.ts b/packages/device-link/src/__tests__/remoteDesktop.test.ts
index 05a6133b75..49d5d76252 100644
--- a/packages/device-link/src/__tests__/remoteDesktop.test.ts
+++ b/packages/device-link/src/__tests__/remoteDesktop.test.ts
@@ -4,6 +4,7 @@ import {
parseRemoteDesktopRequest,
REMOTE_DESKTOP_CHANNEL,
isDesktopPermission,
+ remoteDesktopVideoSettingsWire,
} from "../remoteDesktop";
import { REMOTE_INVOKE_ALLOWLIST, PUSH_FORWARD_ALLOWLIST } from "../allowlist";
describe("remote desktop wire boundary", () => {
@@ -80,4 +81,67 @@ describe("remote desktop wire boundary", () => {
}),
).toThrow();
});
+ describe("video quality tiers", () => {
+ const offer = (settings: unknown) =>
+ parseRemoteDesktopRequest({ op: "offer", lease: "a", sdp: "s", settings });
+ it("keeps only the tier, so hosts own the concrete encoder budget", () => {
+ for (const quality of ["auto", "saver", "hd"] as const)
+ expect(
+ offer(remoteDesktopVideoSettingsWire({ fps: 60, quality, audio: true })),
+ ).toMatchObject({
+ settings: { fps: quality === "saver" ? 30 : 60, quality, audio: true },
+ });
+ expect(
+ offer({ fps: 30, quality: "hd", bitrate: 20_000_000, audio: false }),
+ ).toEqual({
+ op: "offer",
+ lease: "a",
+ sdp: "s",
+ settings: { fps: 30, quality: "hd", audio: false },
+ });
+ });
+ it("maps bitrate-only settings from older viewers onto tiers", () => {
+ for (const [bitrate, quality] of [
+ [0, "auto"],
+ [2_000_000, "saver"],
+ [8_000_000, "hd"],
+ [20_000_000, "hd"],
+ ] as const)
+ expect(offer({ fps: 30, bitrate, audio: false })).toMatchObject({
+ settings: { fps: 30, quality, audio: false },
+ });
+ });
+ it("sends a legacy bitrate that older hosts accept, also for unknown future tiers", () => {
+ expect(
+ [ "auto", "saver", "hd" ].map(
+ (quality) =>
+ remoteDesktopVideoSettingsWire({
+ fps: 30,
+ quality: quality as "auto" | "saver" | "hd",
+ audio: false,
+ }).bitrate,
+ ),
+ ).toEqual([0, 2_000_000, 20_000_000]);
+ expect(
+ offer({ fps: 30, quality: "ultra", bitrate: 20_000_000, audio: false }),
+ ).toMatchObject({ settings: { quality: "hd" } });
+ // Older hosts ignore `quality`, so Saver's 30 fps cap travels in `fps`.
+ expect(
+ remoteDesktopVideoSettingsWire({ fps: 60, quality: "saver", audio: false }),
+ ).toEqual({ fps: 30, quality: "saver", audio: false, bitrate: 2_000_000 });
+ expect(
+ remoteDesktopVideoSettingsWire({ fps: 60, quality: "auto", audio: false }).fps,
+ ).toBe(60);
+ });
+ it.each([
+ { fps: 30, audio: false },
+ { fps: 30, quality: "ultra", audio: false },
+ { fps: 30, bitrate: 5_000_000, audio: false },
+ { fps: 24, quality: "auto", audio: false },
+ { fps: 30, quality: "auto", audio: "yes" },
+ null,
+ ])("rejects invalid settings %j", (settings) => {
+ expect(() => offer(settings)).toThrow("INVALID_REQUEST");
+ });
+ });
});
diff --git a/packages/device-link/src/__tests__/remoteDesktopIce.test.ts b/packages/device-link/src/__tests__/remoteDesktopIce.test.ts
index 25504d0465..7d484d734d 100644
--- a/packages/device-link/src/__tests__/remoteDesktopIce.test.ts
+++ b/packages/device-link/src/__tests__/remoteDesktopIce.test.ts
@@ -23,7 +23,11 @@ it("projects candidate fields and retains both old and incremental offer shapes"
expect(
parseRemoteDesktopRequest({ op: "offer", lease: "lease", sdp: "sdp" }),
).toEqual({ op: "offer", lease: "lease", sdp: "sdp" });
- for (const settings of [undefined, { fps: 30, bitrate: 0, audio: false }])
+ for (const settings of [
+ undefined,
+ { fps: 30, bitrate: 0, audio: false },
+ { fps: 30, quality: "auto", audio: false },
+ ])
expect(
parseRemoteDesktopRequest({
op: "offer",
diff --git a/packages/device-link/src/__tests__/remoteDesktopViewerMedia.test.ts b/packages/device-link/src/__tests__/remoteDesktopViewerMedia.test.ts
index a0e733b172..081570f009 100644
--- a/packages/device-link/src/__tests__/remoteDesktopViewerMedia.test.ts
+++ b/packages/device-link/src/__tests__/remoteDesktopViewerMedia.test.ts
@@ -18,7 +18,7 @@ function viewer(lease: string, request: DesktopViewerRequest) {
canControl: false,
displays: [],
},
- settings: { fps: 30 as const, bitrate: 0 as const, audio: false },
+ settings: { fps: 30 as const, quality: "auto" as const, audio: false },
};
const media = new RemoteDesktopViewerMedia({
request,
@@ -91,3 +91,52 @@ it("does not classify an ordinary video failure as consent and discards a retire
expect(h.send).not.toHaveBeenCalled();
expect(h.onOfferFailure).toHaveBeenCalledOnce();
});
+
+it("sends the tier with a legacy bitrate only to hosts that accept video settings", async () => {
+ const request = vi.fn(async () => ({ sdp: "answer" }));
+ const h = viewer("lease", request as unknown as DesktopViewerRequest);
+ await h.offer();
+ expect(request).toHaveBeenLastCalledWith(
+ expect.not.objectContaining({ settings: expect.anything() }),
+ expect.any(Function),
+ );
+ const caps = { videoSettings: true };
+ const settings = {
+ fps: 60 as const,
+ quality: "saver" as const,
+ audio: false,
+ };
+ const media = new RemoteDesktopViewerMedia({
+ request: request as unknown as DesktopViewerRequest,
+ send: vi.fn(),
+ current: () => ({
+ lease: {
+ lease: "lease",
+ controlling: false,
+ display: { id: "s", name: "S", width: 1, height: 1 },
+ },
+ caps: {
+ version: 1,
+ enabled: true,
+ platform: "darwin",
+ canControl: true,
+ displays: [],
+ ...caps,
+ },
+ settings,
+ }),
+ loadIce: async () => [],
+ });
+ await media.handle({
+ type: "offer",
+ epoch: "lease",
+ attemptId: "attempt",
+ sdp: "offer",
+ });
+ expect(request).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ settings: { ...settings, fps: 30, bitrate: 2_000_000 },
+ }),
+ expect.any(Function),
+ );
+});
diff --git a/packages/device-link/src/remoteDesktop.ts b/packages/device-link/src/remoteDesktop.ts
index 2a81e965de..87c6963705 100644
--- a/packages/device-link/src/remoteDesktop.ts
+++ b/packages/device-link/src/remoteDesktop.ts
@@ -94,11 +94,31 @@ export function isDesktopInput(value: unknown): value is DesktopInput {
}
}
+/** Viewer intent; the host owns the concrete encoder and capture parameters. */
+export type RemoteDesktopVideoQuality = "auto" | "saver" | "hd";
+export const REMOTE_DESKTOP_VIDEO_QUALITIES: readonly RemoteDesktopVideoQuality[] =
+ ["auto", "saver", "hd"];
export interface RemoteDesktopVideoSettings {
fps: 30 | 60;
- bitrate: 0 | 2_000_000 | 8_000_000 | 20_000_000;
+ quality: RemoteDesktopVideoQuality;
audio: boolean;
}
+type LegacyVideoBitrate = 0 | 2_000_000 | 8_000_000 | 20_000_000;
+const LEGACY_BITRATES: readonly number[] = [
+ 0, 2_000_000, 8_000_000, 20_000_000,
+];
+/** Older hosts only accept `bitrate` and ignore `quality`; send both. Saver
+ * also caps the frame rate, which older hosts can only learn from `fps`. */
+export function remoteDesktopVideoSettingsWire(
+ settings: RemoteDesktopVideoSettings,
+): RemoteDesktopVideoSettings & { bitrate: LegacyVideoBitrate } {
+ if (settings.quality === "saver")
+ return { ...settings, fps: 30, bitrate: 2_000_000 };
+ return { ...settings, bitrate: settings.quality === "hd" ? 20_000_000 : 0 };
+}
+function legacyVideoQuality(bitrate: number): RemoteDesktopVideoQuality {
+ return bitrate === 0 ? "auto" : bitrate === 2_000_000 ? "saver" : "hd";
+}
export interface RemoteDesktopDisplayMode {
id: string;
width: number;
@@ -381,11 +401,20 @@ export function parseRemoteDesktopRequest(
};
if (v.settings === undefined)
return { op: v.op, lease, sdp: v.sdp, ...overlay };
- const settings = v.settings as RemoteDesktopVideoSettings;
+ const settings = v.settings as Record<string, unknown> | null;
+ // Older viewers send only `bitrate`; newer ones send `quality` plus a
+ // legacy bitrate, which also covers tiers this host does not know yet.
+ const quality = REMOTE_DESKTOP_VIDEO_QUALITIES.includes(
+ settings?.quality as RemoteDesktopVideoQuality,
+ )
+ ? (settings!.quality as RemoteDesktopVideoQuality)
+ : LEGACY_BITRATES.includes(settings?.bitrate as number)
+ ? legacyVideoQuality(settings!.bitrate as number)
+ : null;
if (
!settings ||
- ![30, 60].includes(settings.fps) ||
- ![0, 2_000_000, 8_000_000, 20_000_000].includes(settings.bitrate) ||
+ (settings.fps !== 30 && settings.fps !== 60) ||
+ !quality ||
typeof settings.audio !== "boolean"
)
throw new Error("INVALID_REQUEST");
@@ -394,11 +423,7 @@ export function parseRemoteDesktopRequest(
lease,
sdp: v.sdp,
...overlay,
- settings: {
- fps: settings.fps,
- bitrate: settings.bitrate,
- audio: settings.audio,
- },
+ settings: { fps: settings.fps, quality, audio: settings.audio },
};
}
if (
diff --git a/packages/device-link/src/remoteDesktopViewerMedia.ts b/packages/device-link/src/remoteDesktopViewerMedia.ts
index 2b2b07cf03..be777930c2 100644
--- a/packages/device-link/src/remoteDesktopViewerMedia.ts
+++ b/packages/device-link/src/remoteDesktopViewerMedia.ts
@@ -4,10 +4,11 @@ import {
parseDesktopIceCandidates,
parseDesktopIceReply,
} from "./remoteDesktopIce.js";
-import type {
- RemoteDesktopCapabilities,
- RemoteDesktopLease,
- RemoteDesktopVideoSettings,
+import {
+ remoteDesktopVideoSettingsWire,
+ type RemoteDesktopCapabilities,
+ type RemoteDesktopLease,
+ type RemoteDesktopVideoSettings,
} from "./remoteDesktop.js";
import type { DesktopViewerRequest } from "./remoteDesktopViewerSession.js";
import type { DesktopIceServer } from "./remoteDesktopIceConfig.js";
@@ -97,7 +98,7 @@ export class RemoteDesktopViewerMedia {
...(current.caps.trickleIce ? { attemptId: attempt } : {}),
cursorOverlay: current.caps.cursorOverlay === true,
...(current.caps.videoSettings
- ? { settings: current.settings }
+ ? { settings: remoteDesktopVideoSettingsWire(current.settings) }
: {}),
},
check,