Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(device-link): bound gap replay after controller reconnect

DavidShenb15ba0162636
diff --git a/docs/dev-rules/remote-and-mobile-adaptation.md b/docs/dev-rules/remote-and-mobile-adaptation.md
index 3f346fbe27..6c5e513646 100644
--- a/docs/dev-rules/remote-and-mobile-adaptation.md
+++ b/docs/dev-rules/remote-and-mobile-adaptation.md
@@ -96,6 +96,14 @@ Mobile 的模型目录变化通知由 `deviceCatalogRefresh.ts` 按设备合并
为 ACK/握手保留硬上限之前的余量;不等待 relay 的 1013 才降速,也不对已排空的健康
socket 固定限速。最大逻辑消息仍可在排空后原子发送,不改变可靠序号和跨版本协议。
+控制端单独重连可能仍使用原可靠流,被控端因自身 relay 未重连而将建链判为重复 open。
+新版确认 ACK 若匹配本次 accept,且明确停在已发送队头的前一序号,可额外补发这一个队头;
+每个序号最多补发一次,同一 peer 的补发间隔不少于原重试基础间隔,且受原恢复帧预算、
+socket 背压和聚合公平预算约束。慢速在途消息也可能得到一次副本,因此不能对每次 open
+重放整队。普通 ACK、旧端无确认能力、已确认数据和过大的队头沿用原重试策略。
+此修复只作用于该 peer,不重启共享 relay、不缩短业务超时,也不改变 wire 格式;
+多控制端故障隔离用例见 `packages/device-link/src/__tests__/client.test.ts`。
+
## 模块通过 Remote Resource 接入移动端
面向移动端新增独立产品入口时,默认通过 `@cindy/device-link` 的 Remote Resource
diff --git a/packages/device-link/src/__tests__/client.test.ts b/packages/device-link/src/__tests__/client.test.ts
index 46b94ce5b2..0719616b97 100644
--- a/packages/device-link/src/__tests__/client.test.ts
+++ b/packages/device-link/src/__tests__/client.test.ts
@@ -9026,3 +9026,111 @@ describe('physical addressing with explicit shared task scope', () => {
} finally { h.client.stop(); }
});
});
+
+
+describe('confirmed duplicate-open gap repair', () => {
+ it('repairs a lost reply after only the controller reconnects without disturbing another controller', async () => {
+ vi.useFakeTimers();
+ const relay = new MemoryRelay();
+ const sockets = vi.spyOn(relay, 'makeWebSocket');
+ const host = makeRelayClient(relay, 'desktop', { transportRetryIntervalMs: 2000 });
+ const phone = makeRelayClient(relay, 'phone', { transportRetryIntervalMs: 2000 });
+ const healthy = makeRelayClient(relay, 'healthy');
+ const pump = () => relay.settle(async () => { await vi.advanceTimersByTimeAsync(1); });
+ const off = host.onFrame((env) => {
+ if (!env.src || !env.id) return;
+ if (env.kind === 'link-open') host.sendLinkAccept(env.src, env.id, { appVersion: '1', allowlistHash: 'hash' });
+ if (env.kind === 'invoke') host.sendInvokeResult(env.src, env.id, {
+ ok: true, result: (env.payload as { channel: string }).channel === 'maker:provider:list' ? 'x'.repeat(100000) : 'fresh',
+ });
+ });
+ try {
+ for (const c of [host, phone, healthy]) c.start();
+ await vi.advanceTimersByTimeAsync(1); await pump();
+ const open = async (c: DeviceLinkClient) => {
+ const p = c.openLink('desktop', { controllerName: 'Viewer', protocolVersion: 1, appVersion: '1' });
+ await pump(); await p;
+ };
+ await open(phone); await open(healthy);
+ relay.dropNext((sender, env) => sender === 'desktop' && env.dst === 'phone' && env.kind === 'invoke-result');
+ const old = phone.invoke('desktop', { channel: 'maker:provider:list', args: [] }, 60000).catch(e => e);
+ await pump(); await vi.advanceTimersByTimeAsync(1000);
+ phone.restartConnection('phone-only-reconnect');
+ await vi.advanceTimersByTimeAsync(1); await pump(); await open(phone);
+ const fresh = phone.invoke('desktop', { channel: 'local-db:sessions:list', args: [] }, 12000);
+ const other = healthy.invoke('desktop', { channel: 'local-db:sessions:list', args: [] }, 12000);
+ await pump();
+ await expect(fresh).resolves.toEqual({ ok: true, result: 'fresh' });
+ await expect(other).resolves.toEqual({ ok: true, result: 'fresh' });
+ expect(host.isLinkReady('healthy')).toBe(true);
+ expect(sockets.mock.calls.filter(x => x[0] === 'desktop')).toHaveLength(1);
+ expect(sockets.mock.calls.filter(x => x[0] === 'healthy')).toHaveLength(1);
+ await old;
+ } finally { off(); for (const c of [host, phone, healthy]) c.stop(); sockets.mockRestore(); vi.useRealTimers(); }
+ });
+
+ async function fixture(bytes = 100000, budget = 8) {
+ const h = makeHarness({ timing: { pingIntervalMs: 600000, transportRetryIntervalMs: 2000, transportRetryPassBudget: budget } });
+ h.client.start(); await vi.advanceTimersByTimeAsync(1); h.current().ack();
+ h.client.onFrame(e => { if (e.kind === 'link-open' && e.id && e.src) h.client.sendLinkAccept(e.src, e.id, { appVersion: '1', allowlistHash: 'hash' }); });
+ const open = async (id: string, confirm = true) => {
+ h.current().push({ v: PROTOCOL_VERSION, kind: 'link-open', id, src: 'phone', payload: {
+ controllerName: 'Viewer', protocolVersion: 1, appVersion: '1', transportStreamId: 'phone-stream', transportBaseSeq: 1,
+ capabilities: [DEVICE_LINK_CAPABILITY_RELIABLE_TRANSPORT, ...(confirm ? [DEVICE_LINK_CAPABILITY_RELIABLE_LINK_CONFIRM] : [])],
+ } });
+ await vi.advanceTimersByTimeAsync(1);
+ };
+ const ack = (id?: string, seq = 0, stream?: string) => {
+ const accept = h.current().sent.filter(e => e.kind === 'link-accept').at(-1)!;
+ h.current().push({ v: PROTOCOL_VERSION, kind: 'push', src: 'phone', payload: {
+ channel: DEVICE_LINK_TRANSPORT_ACK_CHANNEL,
+ payload: { streamId: stream ?? (accept.payload as { transportStreamId: string }).transportStreamId, ackSeq: seq, ...(id ? { linkRequestId: id } : {}) },
+ } });
+ };
+ await open('initial'); ack('initial');
+ h.client.sendInvokeResult('phone', 'head', { ok: true, result: 'x'.repeat(bytes) });
+ const copies = () => h.current().sent.filter(e => e.kind === 'invoke-result' && e.id === 'head').length;
+ return { h, open, ack, copies };
+ }
+
+ it('requires current confirmation and gap, sends only the head once despite duplicate opens and ACKs', async () => {
+ vi.useFakeTimers(); const f = await fixture();
+ try {
+ f.h.client.sendInvokeResult('phone', 'tail', { ok: true, result: 'tail' });
+ await f.open('stale'); await f.open('current');
+ f.ack(); f.ack('stale'); f.ack('current', 0, 'wrong-stream');
+ expect(f.copies()).toBe(1);
+ f.ack('current'); expect(f.copies()).toBe(2);
+ f.ack('current');
+ for (let i=0; i<5; i++) { await f.open('again-'+i); f.ack('again-'+i); }
+ expect(f.copies()).toBe(2);
+ expect(f.h.current().sent.filter(e => e.kind === 'invoke-result' && e.id === 'tail')).toHaveLength(1);
+ } finally { f.h.client.stop(); vi.useRealTimers(); }
+ });
+
+ it.each(['acknowledged','legacy','backpressure','closed','oversize'] as const)('preserves normal retry policy for %s', async mode => {
+ vi.useFakeTimers(); const f = await fixture(mode === 'oversize' ? MAX_TRANSPORT_CHUNK_BYTES * 2 : 100000, mode === 'oversize' ? 1 : 8);
+ try {
+ const before = f.copies(); await f.open('again', mode !== 'legacy');
+ if (mode === 'backpressure') f.h.current().bufferedAmount = MAX_TRANSPORT_WEBSOCKET_BUFFERED_BYTES;
+ if (mode === 'closed') f.h.client.closeLink('phone', 'user');
+ f.ack('again', mode === 'acknowledged' ? 1 : 0);
+ expect(f.copies()).toBe(before);
+ } finally { f.h.client.stop(); vi.useRealTimers(); }
+ });
+
+ it('rate limits repairs across advancing heads without suppressing later eligible gaps', async () => {
+ vi.useFakeTimers(); const f = await fixture();
+ try {
+ await f.open('first'); f.ack('first'); expect(f.copies()).toBe(2);
+ f.ack(undefined, 1);
+ f.h.client.sendInvokeResult('phone','second',{ok:true,result:'x'.repeat(100000)});
+ await f.open('too-soon'); f.ack('too-soon',1);
+ const copies = () => f.h.current().sent.filter(e=>e.id==='second'&&e.kind==='invoke-result').length;
+ expect(copies()).toBe(1);
+ await vi.advanceTimersByTimeAsync(2000);
+ await f.open('later'); f.ack('later',1);
+ expect(copies()).toBe(2);
+ } finally { f.h.client.stop(); vi.useRealTimers(); }
+ });
+});
diff --git a/packages/device-link/src/client.ts b/packages/device-link/src/client.ts
index 8cac2895b5..fbd6b2d1c8 100644
--- a/packages/device-link/src/client.ts
+++ b/packages/device-link/src/client.ts
@@ -608,6 +608,11 @@ interface PeerTransportState {
lastReplayEpoch: number;
/** 上次真正 replay 时对端的 stream。对端重启换 stream 时不能当重复 open。 */
lastReplayRemoteStreamId: string | null;
+ /** Latest duplicate accept whose confirmation may prove a receive gap. */
+ duplicateOpenRepair: { requestId: string; headSeq: number } | null;
+ /** One extra repair per sequence, retained across link resets of this stream. */
+ lastDuplicateOpenRepairSeq: number;
+ lastDuplicateOpenRepairAt: number;
/**
* 恢复探测:link 刚恢复或刚被 DEVICE_OFFLINE 刹停后为 true。
* 此间出站不超过 transportRetryPassBudget,直到收到可靠 ACK。
@@ -1431,6 +1436,7 @@ export class DeviceLinkClient {
);
}
peer.linkAcceptedInbound = true;
+ peer.duplicateOpenRepair = null;
if (peerSupportsReliable) {
const resume = this.planReliableSendResume(peer);
this.commitReliableReceiveReady(dst, peer);
@@ -3407,6 +3413,7 @@ export class DeviceLinkClient {
}
private markPeerLinkDown(peer: PeerTransportState): void {
+ peer.duplicateOpenRepair = null;
if (peer.pendingLinkConfirmation?.timer) {
clearTimeout(peer.pendingLinkConfirmation.timer);
}
@@ -3444,6 +3451,9 @@ export class DeviceLinkClient {
highestAckSeq: 0,
lastReplayEpoch: this.connEpoch,
lastReplayRemoteStreamId: null,
+ duplicateOpenRepair: null,
+ lastDuplicateOpenRepairSeq: 0,
+ lastDuplicateOpenRepairAt: Number.NEGATIVE_INFINITY,
recoveryNeedsAck: false,
recoveryFramesSent: 0,
recoveryTrace: null,
@@ -3561,6 +3571,7 @@ export class DeviceLinkClient {
: 1;
if (peer.remoteStreamId !== nextRemoteStreamId) {
peer.receive.clear();
+ peer.duplicateOpenRepair = null;
}
if (peer.reliable && !reliable) {
this.abandonReliablePending(dst, 'peer no longer supports reliable transport');
@@ -3728,6 +3739,7 @@ export class DeviceLinkClient {
this.commitReliableSendResume(src, peer, confirmation.resume);
}
if (!this.isPeerSendReady(peer)) return;
+ this.repairConfirmedReceiveGap(src, peer, ackSeq, linkRequestId);
// 迟到/陈旧 ACK 幂等无害(含指向已被驱逐 seq 的 ACK):驱逐后该 seq 已不在
// map 里,累计删除循环遇到更高的队头 live seq 直接 break,不会误删、不抛错、
// 不错误推进状态;高于 nextSeq-1 的未知 ACK 与倒退的 ACK 直接忽略。
@@ -3763,6 +3775,42 @@ export class DeviceLinkClient {
}
}
+ /** A controller can reconnect without changing its reliable stream or our
+ * relay epoch. Its current accept confirmation can still report a missing
+ * head. Repair only that head, once per sequence, instead of replaying the
+ * whole queue on every duplicate open. Normal ACKs and legacy peers do not
+ * activate this path. Slow in-flight heads may get one extra copy. */
+ private repairConfirmedReceiveGap(
+ dst: string,
+ peer: PeerTransportState,
+ ackSeq: number,
+ linkRequestId?: string,
+ ): void {
+ const repair = peer.duplicateOpenRepair;
+ if (!repair || repair.requestId !== linkRequestId) return;
+ peer.duplicateOpenRepair = null;
+ const head = peer.pending.values().next().value;
+ const now = this.monotonicNow();
+ const frameCount = head ? this.estimateReliableFrameCount(head) : 0;
+ if (!head?.sent || head.seq !== repair.headSeq
+ || ackSeq !== head.seq - 1 || ackSeq < peer.highestAckSeq
+ || peer.lastDuplicateOpenRepairSeq >= head.seq
+ || now - peer.lastDuplicateOpenRepairAt < normalizeTransportRetryInterval(this.timing.transportRetryIntervalMs)
+ || head.attempts >= this.timing.transportMaxRetryAttempts
+ || frameCount > this.recoveryPassBudget()
+ || this.shouldHoldRecoverySend(peer, frameCount)) return;
+ try {
+ const frames = this.sendReliableFrames(peer, head);
+ if (frames === 0) return; // Existing retry/drain owns local backpressure.
+ peer.lastDuplicateOpenRepairSeq = head.seq;
+ peer.lastDuplicateOpenRepairAt = now;
+ this.noteRecoveryFrames(peer, frames);
+ this.log.debug(`device-link confirmed gap repair dst=${dst.slice(0, 8)} seq=${head.seq} frames=${frames}`);
+ } catch (err) {
+ this.log.debug(`device-link confirmed gap repair deferred dst=${dst.slice(0, 8)}`, err);
+ }
+ }
+
/**
* 单调时钟。pending 帧的滞留时长(TTL)必须用不受墙钟校正影响的时源计量:
* Date.now() 在系统时间被向前校正超过 TRANSPORT_PENDING_PUSH_MAX_AGE_MS 时,
@@ -4155,6 +4203,8 @@ export class DeviceLinkClient {
// 同 stream 重复 open:发送方向已经 ready 时绝不能再打回 awaiting-confirm,
// 否则 ACK/重试被暂停,pending 只涨不消,对端超时后再 open,形成死循环。
if (resume.duplicateOpen && this.isPeerSendReady(peer)) {
+ const head = peer.pending.values().next().value;
+ peer.duplicateOpenRepair = head?.sent ? { requestId, headSeq: head.seq } : null;
// sendLinkAccept 已把这次 accept 记进 active 路由账本;确认分支才会
// discard。这里保持 ready、不进 awaiting-confirm,但仍要结算该记录,
// 否则每次重复 open 泄漏一条,满 1024 后后续 accept 全 BACKPRESSURE。