Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(device-link): keep relay fallback progressing in sequence

MagicLizibf0ce0032dc6
diff --git a/docs/dev-rules/remote-and-mobile-adaptation.md b/docs/dev-rules/remote-and-mobile-adaptation.md
index 3789927413..ad8b46e112 100644
--- a/docs/dev-rules/remote-and-mobile-adaptation.md
+++ b/docs/dev-rules/remote-and-mobile-adaptation.md
@@ -123,8 +123,9 @@ relay 以 close 1013 `inbound backpressure` 主动断连,此时任何「立即
被控开关、撤权名单与 allowlist。共享任务、供应商分享等跨账号范围不走直连,逐帧
授权仍由 relay 执行。relay 报告对端离线(含被移出账号)、链路被关闭或撤权、客户端
停止时立即撤下直连。ACK 原路返回,单向失效时发送端不会被困在直连上。
-- **故障半径三问**:① 故障层级是单个 peer 的直连通道(通道关闭、写入失败、经直连
- 发出的帧过了重发期限仍未确认);② 动作只撤下这一台设备的直连,把它仍未确认的帧
+- **故障半径三问**:① 故障层级是单个 peer 的直连通道(通道关闭、写入失败、发送缓冲
+ 在一个可靠重试基准期内没有下降,或已发出的帧过了重发期限仍未确认);缓冲仍有进展时
+ 继续等待排空,不误判正常背压;② 动作只撤下这一台设备的直连,把它仍未确认的帧
恢复为未发送、按 relay 既有恢复预算(先探测一批、收到 ACK 再继续)重发,不重连
relay、不重建链路,直连的失败不消耗 relay 的重试次数;③ 回归见
`packages/device-link/src/__tests__/client.test.ts`「直连优先、中继兜底」:两个控制端
diff --git a/packages/device-link/src/__tests__/client.test.ts b/packages/device-link/src/__tests__/client.test.ts
index c8116bd74b..9a499f0335 100644
--- a/packages/device-link/src/__tests__/client.test.ts
+++ b/packages/device-link/src/__tests__/client.test.ts
@@ -39,10 +39,11 @@ import { resolveRemoteInvokeTimeoutMs } from '../invokePolicy.js';
import { SESSION_ACTIVITY_CHANNEL } from '../topics.js';
import { resolveDesktopIceServers, REMOTE_DESKTOP_ICE_CONFIG_TIMEOUT_MS } from '../remoteDesktopIceConfig.js';
import { REMOTE_DESKTOP_ICE_SERVERS } from '../remoteDesktopIce.js';
-import type {
- DeviceLinkDirectChannel,
- DeviceLinkDirectPathHandle,
- DeviceLinkPeerPathChanged,
+import {
+ DIRECT_PATH_HIGH_WATER_BYTES,
+ type DeviceLinkDirectChannel,
+ type DeviceLinkDirectPathHandle,
+ type DeviceLinkPeerPathChanged,
} from '../directPath.js';
type Handler = (...args: unknown[]) => void;
@@ -10498,6 +10499,7 @@ function connectDirect(
) {
const drop = { toLeft: false, toRight: false };
const failSend = { left: false, right: false };
+ const buffered = { left: 0, right: 0 };
const frames = { toLeft: [] as Envelope[], toRight: [] as Envelope[] };
const closeReasons: string[] = [];
let open = true;
@@ -10506,6 +10508,7 @@ function connectDirect(
const direction = from === 'left' ? 'toRight' : 'toLeft';
const target = from === 'left' ? 'right' : 'left';
return {
+ get bufferedAmount() { return buffered[from]; },
send(text) {
if (!open || failSend[from]) throw new Error('direct channel unavailable');
frames[direction].push(JSON.parse(text) as Envelope);
@@ -10521,13 +10524,150 @@ function connectDirect(
};
handles.left = left.client.attachDirectPath(right.id, channel('left'));
handles.right = right.client.attachDirectPath(left.id, channel('right'));
- return { drop, failSend, frames, closeReasons, handles, isOpen: () => open };
+ return { drop, failSend, buffered, frames, closeReasons, handles, isOpen: () => open };
}
const reliableKinds = (envs: readonly Envelope[] | undefined, kind: Envelope['kind'], src?: string): number =>
(envs ?? []).filter(env => env.kind === kind && parseTransportPayload(env.payload) && (!src || env.src === src)).length;
describe('直连优先、中继兜底(Device Link direct path)', () => {
+ it('replays a partially written large head while an older relay tail is still in flight', async () => {
+ vi.useFakeTimers();
+ const h = makeHarness({ timing: { pingIntervalMs: 1_000_000, transportRetryIntervalMs: 1_000 } });
+ try {
+ h.client.start(); await vi.advanceTimersByTimeAsync(0); h.current().ack();
+ const opening = establishInboundReliableLink(h, 'peer-stream');
+ await vi.advanceTimersByTimeAsync(0); await opening;
+ h.client.sendInvokeResult('dev-b', 'head', {
+ ok: true, result: 'x'.repeat(MAX_TRANSPORT_CHUNK_BYTES * (TRANSPORT_RETRY_PASS_BUDGET + 1)),
+ });
+ h.client.sendInvokeResult('dev-b', 'tail', {
+ ok: true, result: 'y'.repeat(MAX_TRANSPORT_CHUNK_BYTES * (TRANSPORT_RETRY_PASS_BUDGET + 4)),
+ });
+ const initial = h.current().sent.slice();
+ const initialHead = initial.filter(env => env.id === 'head');
+ let writes = 0;
+ h.client.attachDirectPath('dev-b', {
+ send() { if (++writes > TRANSPORT_RETRY_PASS_BUDGET) throw new Error('direct write interrupted'); },
+ close() {},
+ });
+ await vi.advanceTimersByTimeAsync(61_000);
+ expect(writes).toBe(TRANSPORT_RETRY_PASS_BUDGET + 1);
+ expect(h.client.getPeerPath('dev-b')).toBe('relay');
+ const fallbackHead = h.current().sent.slice(initial.length).filter(env => env.id === 'head');
+ expect(fallbackHead).toHaveLength(initialHead.length);
+ // Reuse the original sequence and every fragment; the receiver can assemble and deduplicate it.
+ expect(fallbackHead.map(env => parseTransportPayload(env.payload)?.meta))
+ .toEqual(initialHead.map(env => parseTransportPayload(env.payload)?.meta));
+ } finally { h.client.stop(); vi.useRealTimers(); }
+ });
+
+ // A small tail can reach its retry deadline while a fragmented head is still cooling down.
+ it.each(['ack-timeout', 'send-failed'])('replays from the large head when a later direct result triggers %s', async (trigger) => {
+ vi.useFakeTimers();
+ const relay = new MemoryRelay();
+ const host = makeRelayClient(relay, 'desktop', { transportRetryIntervalMs: 1_000, requestTimeoutMs: 30_000 });
+ const a = makeRelayClient(relay, 'viewer-a', { requestTimeoutMs: 30_000 });
+ const pump = () => relay.settle(async () => { await vi.advanceTimersByTimeAsync(1); });
+ const large = 'x'.repeat(MAX_TRANSPORT_CHUNK_BYTES * (TRANSPORT_RETRY_PASS_BUDGET + 1));
+ const results: unknown[] = [];
+ const request = (index: number) => {
+ void a.invoke('desktop', { channel: 'local-db:sessions:get', args: [String(index)] }).then(
+ value => { results[index] = value; }, error => { results[index] = error; },
+ );
+ };
+ let executions = 0;
+ host.onFrame((env) => {
+ if (env.kind === 'link-open') host.sendLinkAccept(env.src!, env.id!, { appVersion: '1', allowlistHash: 'hash' });
+ if (env.kind === 'invoke') {
+ executions++;
+ host.sendInvokeResult(env.src!, env.id!, { ok: true, result: executions === 1 ? large : 'small' });
+ }
+ });
+ try {
+ host.start(); a.start();
+ await pump();
+ const opening = a.openLink('desktop', { controllerName: 'Viewer', protocolVersion: 1, appVersion: '1' });
+ await pump(); await opening;
+ const direct = connectDirect({ id: 'viewer-a', client: a }, { id: 'desktop', client: host });
+ direct.drop.toLeft = true;
+ for (let index = 0; index < 2; index++) request(index);
+ await pump();
+ expect(executions).toBe(2);
+ expect(results).toHaveLength(0);
+ if (trigger === 'send-failed') {
+ direct.failSend.right = true;
+ request(2);
+ } else {
+ await vi.advanceTimersByTimeAsync(1_100);
+ }
+ await pump();
+ const expectedCount = trigger === 'send-failed' ? 3 : 2;
+ expect(results).toHaveLength(expectedCount);
+ expect(results[0]).toEqual({ ok: true, result: large });
+ for (const result of results.slice(1)) expect(result).toEqual({ ok: true, result: 'small' });
+ expect(executions).toBe(expectedCount);
+ expect(host.getPeerPath('viewer-a')).toBe('relay');
+ } finally {
+ host.stop(); a.stop();
+ vi.useRealTimers();
+ }
+ });
+
+ // A healthy relay must remain usable even when no direct frame ever gets its first write.
+ it.each([false, true])('handles direct buffer pressure before the first write (stalled=%s)', async (stalled) => {
+ vi.useFakeTimers();
+ const relay = new MemoryRelay();
+ const timing = { transportRetryIntervalMs: 1_000, requestTimeoutMs: 30_000 };
+ const host = makeRelayClient(relay, 'desktop', timing);
+ const a = makeRelayClient(relay, 'viewer-a', timing);
+ const pump = () => relay.settle(async () => { await vi.advanceTimersByTimeAsync(1); });
+ let executions = 0;
+ host.onFrame((env) => {
+ if (env.kind === 'link-open') host.sendLinkAccept(env.src!, env.id!, { appVersion: '1', allowlistHash: 'hash' });
+ if (env.kind === 'invoke') {
+ executions++;
+ host.sendInvokeResult(env.src!, env.id!, { ok: true, result: 'ok' });
+ }
+ });
+ try {
+ host.start(); a.start();
+ await pump();
+ const opening = a.openLink('desktop', { controllerName: 'Viewer', protocolVersion: 1, appVersion: '1' });
+ await pump(); await opening;
+ const direct = connectDirect({ id: 'viewer-a', client: a }, { id: 'desktop', client: host });
+ direct.buffered.left = DIRECT_PATH_HIGH_WATER_BYTES;
+ let result: unknown;
+ void a.invoke('desktop', { channel: 'local-db:sessions:get', args: ['s1'] }).then(
+ value => { result = value; }, error => { result = error; },
+ );
+ await pump();
+ expect(executions).toBe(0);
+ expect(result).toBeUndefined();
+ expect(a.getPeerPath('desktop')).toBe('direct');
+ await vi.advanceTimersByTimeAsync(500);
+ if (stalled) {
+ // Bytes draining slowly are progress: don't turn a healthy slow channel into a timeout.
+ direct.buffered.left--;
+ direct.handles.left!.drained();
+ await vi.advanceTimersByTimeAsync(600);
+ expect(a.getPeerPath('desktop')).toBe('direct');
+ await vi.advanceTimersByTimeAsync(1_000);
+ } else {
+ direct.buffered.left = 0;
+ direct.handles.left!.drained();
+ }
+ await pump();
+ expect(result).toEqual({ ok: true, result: 'ok' });
+ expect(executions).toBe(1);
+ expect(a.getPeerPath('desktop')).toBe(stalled ? 'relay' : 'direct');
+ expect(reliableKinds(relay.deliveredTo.get('desktop'), 'invoke')).toBe(stalled ? 1 : 0);
+ } finally {
+ host.stop(); a.stop();
+ vi.useRealTimers();
+ }
+ });
+
it('falls back to relay on a stalled direct path without re-running the command or disturbing another controller', async () => {
vi.useFakeTimers();
const relay = new MemoryRelay();
diff --git a/packages/device-link/src/client.ts b/packages/device-link/src/client.ts
index 6b5521cc6e..1222294cd1 100644
--- a/packages/device-link/src/client.ts
+++ b/packages/device-link/src/client.ts
@@ -700,6 +700,8 @@ interface PeerTransportState {
* relay 时,ACK 不能继续投进可能已失效的直连通道。
*/
lastInboundPath: DeviceLinkPeerPath;
+ /** 直连尚未写出时的缓冲进展;持续不下降也要切回 relay,不能只等待已发送帧的 ACK。 */
+ directBufferWait?: { channel: DeviceLinkDirectChannel; bytes: number; since: number };
}
interface PendingInboundLinkOffer {
@@ -3345,6 +3347,10 @@ export class DeviceLinkClient {
const sentDirect = this.sendReliableFramesDirect(peer, pending, frames, wireBytes, direct);
// null: 一帧都没写进直连且通道已撤下,本次立即改走 relay。
if (sentDirect !== null) return sentDirect;
+ // 撤下通道可能把更早的 direct 帧也恢复成未发送。先由 detach 安排的恢复
+ // 从队头推进 ACK,不能让当前短尾帧抢占 relay 探测预算,卡住缺失的大队头。
+ const head = peer.pending.values().next().value;
+ if (head && head !== pending && !head.sent) return 0;
}
const peers = this.getReadyReliablePeers(pending.envelope.dst!);
const budgetNow = this.monotonicNow();
@@ -3437,7 +3443,7 @@ export class DeviceLinkClient {
/**
* 经直连写出一条可靠消息的全部分片。返回 null 表示一帧都没写出且通道已撤下,
- * 调用方立即改走 relay;返回 0 表示直连缓冲已满,帧留在队列等 drained()。
+ * 调用方改走 relay;返回 0 表示暂缓发送或部分写入后已安排从队头恢复。
*/
private sendReliableFramesDirect(
peer: PeerTransportState,
@@ -3450,9 +3456,21 @@ export class DeviceLinkClient {
const buffered = channel.bufferedAmount ?? 0;
// 与 relay 一致:缓冲排空后单条大消息总能发出,不会被软上限永久卡住。
if (buffered > 0 && buffered + wireBytes > DIRECT_PATH_HIGH_WATER_BYTES) {
+ const now = this.monotonicNow();
+ const waiting = peer.directBufferWait;
+ if (!waiting || waiting.channel !== channel || buffered < waiting.bytes) {
+ peer.directBufferWait = { channel, bytes: buffered, since: now };
+ } else {
+ waiting.bytes = buffered;
+ if (now - waiting.since >= retransmitBaseMs(peer.rtt, this.timing.transportRetryIntervalMs)) {
+ this.detachDirectPath(dst, 'buffer-timeout');
+ return null;
+ }
+ }
pending.lastBlockedBy = 'direct-buffer';
return 0;
}
+ peer.directBufferWait = undefined;
let sent = 0;
for (const frame of frames) {
try {
@@ -3470,7 +3488,11 @@ export class DeviceLinkClient {
if (pending.envelope.id) this.pending.get(pending.envelope.id)?.noteFirstWrite?.();
this.noteReliableWrite(peer, pending, sent, 'direct');
// 只写出部分分片:整条消息改由 relay 从头重发,接收端丢弃重复分片。
- if (sent < frames.length) this.detachDirectPath(dst, 'send-failed');
+ if (sent < frames.length) {
+ this.detachDirectPath(dst, 'send-failed');
+ // 这些片已属于撤下的直连,不能再计入新一轮 relay 的恢复预算。
+ return 0;
+ }
return sent;
}
@@ -3487,6 +3509,7 @@ export class DeviceLinkClient {
const peer = this.peerTransport.get(deviceId);
let resent = 0;
if (peer) {
+ peer.directBufferWait = undefined;
peer.lastInboundPath = 'relay';
for (const pending of peer.pending.values()) {
if (pending.path !== 'direct') continue;
@@ -3501,7 +3524,7 @@ export class DeviceLinkClient {
}
}
const message = `device-link direct path closed dst=${deviceId.slice(0, 8)} reason=${reason} resendViaRelay=${resent}`;
- if (reason === 'ack-timeout' || reason === 'send-failed') this.log.warn(message);
+ if (reason === 'ack-timeout' || reason === 'buffer-timeout' || reason === 'send-failed') this.log.warn(message);
else this.log.info(message);
this.emitPeerPathChanged({ deviceId, path: 'relay', reason, ...(resent > 0 ? { resent } : {}) });
if (resent === 0) return;
@@ -3930,6 +3953,7 @@ export class DeviceLinkClient {
peer.receiveReady = false;
peer.pendingLinkConfirmation = null;
peer.outboundLinkConfirmationAck = null;
+ peer.directBufferWait = undefined;
peer.lastInboundPath = 'relay';
}
@@ -4675,13 +4699,18 @@ export class DeviceLinkClient {
// 经直连写出的帧到了重发期限仍未确认:直连已不可信,撤下并改走 relay。
// 这些帧恢复为未发送、重试次数从零计,直连的失败不消耗 relay 的重试预算。
this.detachDirectPath(dst, 'ack-timeout');
+ // 本轮可能在非队头短帧处超时;让 detach 的微任务从最早未确认帧重启恢复。
+ return;
}
if (pending.attempts >= this.timing.transportMaxRetryAttempts) {
this.handleReliableRetryExhausted(dst, pending.seq);
return;
}
const admittingNew = !pending.sent;
- if (admittingNew && this.shouldHoldRecoverySend(peer, this.estimateReliableFrameCount(pending))) {
+ // 回退后队头可能尚未发送,而后面的 relay 帧仍在途。累计 ACK 不能越过
+ // 缺失队头,尾帧不可能释放预算;必须允许队头整条发出,再等 ACK 推进。
+ if (admittingNew && pending !== head
+ && this.shouldHoldRecoverySend(peer, this.estimateReliableFrameCount(pending))) {
pending.lastBlockedBy = 'recovery-ack';
break;
}
diff --git a/packages/device-link/src/directPath.ts b/packages/device-link/src/directPath.ts
index f9bde44b6a..5185d350d1 100644
--- a/packages/device-link/src/directPath.ts
+++ b/packages/device-link/src/directPath.ts
@@ -35,6 +35,8 @@ export type DeviceLinkDirectCloseReason =
| 'send-failed'
/** 经直连发出的帧过了重发期限仍未确认。 */
| 'ack-timeout'
+ /** 直连发送缓冲持续无进展,尚未写出的帧也改走 relay。 */
+ | 'buffer-timeout'
/** relay 报告对端已离线;在线状态以 relay 为准。 */
| 'peer-offline'
/** 控制链路被显式关闭或撤权。 */
@@ -67,5 +69,5 @@ export interface DeviceLinkPeerPathChanged {
resent?: number;
}
-/** 直连发送缓冲软上限:超过后帧留在可靠队列,等 `drained()` 再发。 */
+/** 直连发送缓冲软上限:暂留可靠队列等 `drained()`,持续无进展则回退 relay。 */
export const DIRECT_PATH_HIGH_WATER_BYTES = 1024 * 1024;