Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(device-link): advance bounded gap repair on cumulative acknowledgements

DavidShen7f9331cbe725
diff --git a/docs/dev-rules/remote-and-mobile-adaptation.md b/docs/dev-rules/remote-and-mobile-adaptation.md
index f530f70c63..dbd7058844 100644
--- a/docs/dev-rules/remote-and-mobile-adaptation.md
+++ b/docs/dev-rules/remote-and-mobile-adaptation.md
@@ -118,10 +118,13 @@ Mobile 的模型目录变化通知由 `deviceCatalogRefresh.ts` 按设备合并
socket 固定限速。最大逻辑消息仍可在排空后原子发送,不改变可靠序号和跨版本协议。
控制端单独重连可能仍使用原可靠流,被控端因自身 relay 未重连而将建链判为重复 open。
-新版确认 ACK 若匹配本次 accept,且明确停在已发送队头的前一序号,可额外补发这一个队头;
-每个序号最多补发一次,同一 peer 的补发间隔不少于原重试基础间隔,且受原恢复帧预算、
-socket 背压和聚合公平预算约束。慢速在途消息也可能得到一次副本,因此不能对每次 open
-重放整队。普通 ACK、旧端无确认能力、已确认数据和过大的队头沿用原重试策略。
+新版确认 ACK 若匹配本次 accept,且明确停在已发送队头的前一序号,可额外补发这个队头;
+该队头被累计 ACK 确认后,才能继续补发下一队头,边界固定为本次 accept 时连续已发送的
+积压前缀。重复 open 不延长已启动的补发范围,新入队或当时未发出的消息不纳入。
+每个序号最多补发一次;新一轮补发仍受原重试基础间隔限制,轮内仅由真实 ACK 前进驱动,
+且受原恢复帧预算、socket 背压和聚合公平预算约束。慢速在途消息也可能得到一次副本,
+因此不能对每次 open 重放整队。普通 ACK 不能启动补发;旧端无确认能力、已确认数据、
+过大的队头及本地背压仍沿用原重试策略,断链和换流取消本轮补发。
此修复只作用于该 peer,不重启共享 relay、不缩短业务超时,也不改变 wire 格式;
多控制端故障隔离用例见 `packages/device-link/src/__tests__/client.test.ts`。
diff --git a/packages/device-link/src/__tests__/client.test.ts b/packages/device-link/src/__tests__/client.test.ts
index 26f240bab9..efe7f3d99f 100644
--- a/packages/device-link/src/__tests__/client.test.ts
+++ b/packages/device-link/src/__tests__/client.test.ts
@@ -9154,8 +9154,8 @@ describe('confirmed duplicate-open gap repair', () => {
await pump(); await p;
};
await open(phone); await open(healthy);
- relay.dropNext((sender, env) => sender === 'desktop' && env.dst === 'phone' && env.kind === 'invoke-result');
- const old = phone.invoke('desktop', { channel: 'maker:provider:list', args: [] }, 60000).catch(e => e);
+ for (let i = 0; i < 10; i++) relay.dropNext((sender, env) => sender === 'desktop' && env.dst === 'phone' && env.kind === 'invoke-result');
+ const old = Promise.all(Array.from({ length: 10 }, () => phone.invoke('desktop', { channel: 'maker:provider:list', args: [] }, 60000).catch(e => e)));
await pump(); await vi.advanceTimersByTimeAsync(1000);
phone.restartConnection('phone-only-reconnect');
await vi.advanceTimersByTimeAsync(1); await pump(); await open(phone);
@@ -9167,7 +9167,7 @@ describe('confirmed duplicate-open gap repair', () => {
expect(host.isLinkReady('healthy')).toBe(true);
expect(sockets.mock.calls.filter(x => x[0] === 'desktop')).toHaveLength(1);
expect(sockets.mock.calls.filter(x => x[0] === 'healthy')).toHaveLength(1);
- await old;
+ expect(await old).toEqual(Array.from({ length: 10 }, () => ({ ok: true, result: 'x'.repeat(100000) })));
} finally { off(); for (const c of [host, phone, healthy]) c.stop(); sockets.mockRestore(); vi.useRealTimers(); }
});
@@ -9235,4 +9235,55 @@ describe('confirmed duplicate-open gap repair', () => {
expect(copies()).toBe(2);
} finally { f.h.client.stop(); vi.useRealTimers(); }
});
+
+ it('continues the captured backlog only after each repaired head is acknowledged', async () => {
+ vi.useFakeTimers(); const f = await fixture();
+ try {
+ for (let i = 2; i <= 10; i++) f.h.client.sendInvokeResult('phone', `old-${i}`, { ok: true, result: 'old' });
+ await f.open('reconnect'); f.ack('reconnect');
+ const copies = (id: string) => f.h.current().sent.filter(e => e.kind === 'invoke-result' && e.id === id).length;
+ f.h.client.sendInvokeResult('phone', 'new', { ok: true, result: 'new' });
+ expect(copies('old-2')).toBe(1);
+ // A duplicate open must not extend the recovery snapshot to newly queued work.
+ await f.open('again'); f.ack('again');
+ for (let seq = 1; seq < 10; seq++) {
+ f.ack(undefined, seq);
+ expect(copies(`old-${seq + 1}`)).toBe(2);
+ f.ack(undefined, seq);
+ expect(copies(`old-${seq + 1}`)).toBe(2);
+ if (seq < 9) expect(copies(`old-${seq + 2}`)).toBe(1);
+ }
+ f.ack(undefined, 10);
+ expect(copies('new')).toBe(1);
+ } finally { f.h.client.stop(); vi.useRealTimers(); }
+ });
+
+ it.each(['backpressure', 'oversize', 'closed'] as const)('stops ACK-paced repair on %s', async mode => {
+ vi.useFakeTimers(); const f = await fixture(100000, 1);
+ try {
+ f.h.client.sendInvokeResult('phone', 'tail', { ok: true, result: 'x'.repeat(mode === 'oversize' ? MAX_TRANSPORT_CHUNK_BYTES * 2 : 100) });
+ await f.open('again'); f.ack('again');
+ const copies = () => f.h.current().sent.filter(e => e.kind === 'invoke-result' && e.id === 'tail').length;
+ const before = copies();
+ if (mode === 'backpressure') f.h.current().bufferedAmount = MAX_TRANSPORT_WEBSOCKET_BUFFERED_BYTES;
+ if (mode === 'closed') f.h.client.closeLink('phone', 'user');
+ f.ack(undefined, 1);
+ expect(copies()).toBe(before);
+ } finally { f.h.client.stop(); vi.useRealTimers(); }
+ });
+
+ it('does not include an unsent tail in the captured repair prefix', async () => {
+ vi.useFakeTimers(); const f = await fixture();
+ try {
+ f.h.current().bufferedAmount = 1024 * 1024;
+ f.h.client.sendInvokeResult('phone', 'unsent', { ok: true, result: 'queued' });
+ f.h.current().bufferedAmount = 0;
+ await f.open('again'); f.ack('again');
+ f.ack(undefined, 1);
+ const copies = () => f.h.current().sent.filter(e => e.kind === 'invoke-result' && e.id === 'unsent').length;
+ expect(copies()).toBe(1);
+ f.ack(undefined, 1);
+ expect(copies()).toBe(1);
+ } finally { f.h.client.stop(); vi.useRealTimers(); }
+ });
});
diff --git a/packages/device-link/src/client.ts b/packages/device-link/src/client.ts
index 9f2f802090..ab496ad1bc 100644
--- a/packages/device-link/src/client.ts
+++ b/packages/device-link/src/client.ts
@@ -609,7 +609,9 @@ interface PeerTransportState {
/** 上次真正 replay 时对端的 stream。对端重启换 stream 时不能当重复 open。 */
lastReplayRemoteStreamId: string | null;
/** Latest duplicate accept whose confirmation may prove a receive gap. */
- duplicateOpenRepair: { requestId: string; headSeq: number } | null;
+ duplicateOpenRepair: { requestId: string; headSeq: number; throughSeq: number } | null;
+ /** ACK-paced repair, bounded to the backlog captured by the accepted open. */
+ confirmedGapRepair: { awaitingSeq: number; throughSeq: number } | null;
/** One extra repair per sequence, retained across link resets of this stream. */
lastDuplicateOpenRepairSeq: number;
lastDuplicateOpenRepairAt: number;
@@ -3433,6 +3435,7 @@ export class DeviceLinkClient {
private markPeerLinkDown(peer: PeerTransportState): void {
peer.duplicateOpenRepair = null;
+ peer.confirmedGapRepair = null;
if (peer.pendingLinkConfirmation?.timer) {
clearTimeout(peer.pendingLinkConfirmation.timer);
}
@@ -3471,6 +3474,7 @@ export class DeviceLinkClient {
lastReplayEpoch: this.connEpoch,
lastReplayRemoteStreamId: null,
duplicateOpenRepair: null,
+ confirmedGapRepair: null,
lastDuplicateOpenRepairSeq: 0,
lastDuplicateOpenRepairAt: Number.NEGATIVE_INFINITY,
recoveryNeedsAck: false,
@@ -3591,6 +3595,7 @@ export class DeviceLinkClient {
if (peer.remoteStreamId !== nextRemoteStreamId) {
peer.receive.clear();
peer.duplicateOpenRepair = null;
+ peer.confirmedGapRepair = null;
}
if (peer.reliable && !reliable) {
this.abandonReliablePending(dst, 'peer no longer supports reliable transport');
@@ -3774,11 +3779,16 @@ export class DeviceLinkClient {
peer.pendingBytes -= pending.bytes;
}
if (peer.recoveryNeedsAck) {
+ peer.confirmedGapRepair = null;
peer.recoveryNeedsAck = false;
peer.recoveryFramesSent = 0;
this.logRecoveryStage(src, 'recovery-probe-acked', ` ack=${ackSeq}`);
this.retryPending(src, { ignoreInterval: true });
} else {
+ const repair = peer.confirmedGapRepair;
+ if (repair && ackSeq >= repair.awaitingSeq) {
+ this.sendConfirmedGapHead(src, peer, repair.throughSeq);
+ }
// Release locally queued first sends promptly, without replaying already
// in-flight large responses every time a partial ACK arrives.
this.retryPending(src, { ignoreInterval: false, onlyUnsent: true });
@@ -3796,9 +3806,9 @@ export class DeviceLinkClient {
/** A controller can reconnect without changing its reliable stream or our
* relay epoch. Its current accept confirmation can still report a missing
- * head. Repair only that head, once per sequence, instead of replaying the
- * whole queue on every duplicate open. Normal ACKs and legacy peers do not
- * activate this path. Slow in-flight heads may get one extra copy. */
+ * head. Repair one head at a time, advancing only after its cumulative ACK,
+ * bounded to the captured backlog and one extra copy per sequence. Normal
+ * ACKs and legacy peers cannot start this path. Slow heads may get a copy. */
private repairConfirmedReceiveGap(
dst: string,
peer: PeerTransportState,
@@ -3808,13 +3818,23 @@ export class DeviceLinkClient {
const repair = peer.duplicateOpenRepair;
if (!repair || repair.requestId !== linkRequestId) return;
peer.duplicateOpenRepair = null;
+ if (peer.confirmedGapRepair) return; // Repeated opens cannot extend the snapshot.
const head = peer.pending.values().next().value;
const now = this.monotonicNow();
- const frameCount = head ? this.estimateReliableFrameCount(head) : 0;
if (!head?.sent || head.seq !== repair.headSeq
|| ackSeq !== head.seq - 1 || ackSeq < peer.highestAckSeq
+ || now - peer.lastDuplicateOpenRepairAt < normalizeTransportRetryInterval(this.timing.transportRetryIntervalMs)) return;
+ this.sendConfirmedGapHead(dst, peer, repair.throughSeq);
+ }
+
+ private sendConfirmedGapHead(dst: string, peer: PeerTransportState, throughSeq: number): void {
+ // A blocked/oversize head falls back to the existing retry/drain policy.
+ // It must not allow a later ACK or duplicate open to spend a fresh budget.
+ peer.confirmedGapRepair = null;
+ const head = peer.pending.values().next().value;
+ const frameCount = head ? this.estimateReliableFrameCount(head) : 0;
+ if (!head?.sent || head.seq > throughSeq
|| peer.lastDuplicateOpenRepairSeq >= head.seq
- || now - peer.lastDuplicateOpenRepairAt < normalizeTransportRetryInterval(this.timing.transportRetryIntervalMs)
|| head.attempts >= this.timing.transportMaxRetryAttempts
|| frameCount > this.recoveryPassBudget()
|| this.shouldHoldRecoverySend(peer, frameCount)) return;
@@ -3822,7 +3842,8 @@ export class DeviceLinkClient {
const frames = this.sendReliableFrames(peer, head);
if (frames === 0) return; // Existing retry/drain owns local backpressure.
peer.lastDuplicateOpenRepairSeq = head.seq;
- peer.lastDuplicateOpenRepairAt = now;
+ peer.lastDuplicateOpenRepairAt = this.monotonicNow();
+ if (head.seq < throughSeq) peer.confirmedGapRepair = { awaitingSeq: head.seq, throughSeq };
this.noteRecoveryFrames(peer, frames);
this.log.debug(`device-link confirmed gap repair dst=${dst.slice(0, 8)} seq=${head.seq} frames=${frames}`);
} catch (err) {
@@ -4223,7 +4244,12 @@ export class DeviceLinkClient {
// 否则 ACK/重试被暂停,pending 只涨不消,对端超时后再 open,形成死循环。
if (resume.duplicateOpen && this.isPeerSendReady(peer)) {
const head = peer.pending.values().next().value;
- peer.duplicateOpenRepair = head?.sent ? { requestId, headSeq: head.seq } : null;
+ let throughSeq = 0;
+ for (const pending of peer.pending.values()) {
+ if (!pending.sent) break;
+ throughSeq = pending.seq;
+ }
+ peer.duplicateOpenRepair = head?.sent ? { requestId, headSeq: head.seq, throughSeq } : null;
// sendLinkAccept 已把这次 accept 记进 active 路由账本;确认分支才会
// discard。这里保持 ready、不进 awaiting-confirm,但仍要结算该记录,
// 否则每次重复 open 泄漏一条,满 1024 后后续 accept 全 BACKPRESSURE。
@@ -4304,6 +4330,7 @@ export class DeviceLinkClient {
}
peer.lastReplayEpoch = this.connEpoch;
peer.lastReplayRemoteStreamId = peer.remoteStreamId;
+ peer.confirmedGapRepair = null;
// 真正恢复不依赖当时队列是否有积压:abandon / transport-timeout 可能已清空
// pending。首次建链(还没 resume 过)保持原语义,空队列不进探测。
if (resume.enterRecovery) {