Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(remote-desktop): 穿透状态绑定到批次实际注入时刻

Dashfe8f1ee160fc
diff --git a/apps/desktop/src/main/remote-desktop/__tests__/controlledOverlay.test.ts b/apps/desktop/src/main/remote-desktop/__tests__/controlledOverlay.test.ts
index 7159e0d469..2aaf60cd82 100644
--- a/apps/desktop/src/main/remote-desktop/__tests__/controlledOverlay.test.ts
+++ b/apps/desktop/src/main/remote-desktop/__tests__/controlledOverlay.test.ts
@@ -373,7 +373,7 @@ it('asks only the local user: hidden from capture, passed through, never focused
expect(confirmation.getBounds()).toEqual({ x: 520, y: 388, width: 400, height: 150 });
// The viewer's pointer passes through it as through the overlay.
- overlay.viewerPointer({ x: 700, y: 450 });
+ overlay.viewerPointer([{ x: 700, y: 450 }]);
expect(confirmation.setIgnoreMouseEvents).toHaveBeenLastCalledWith(true);
expect(state.windows[0].setIgnoreMouseEvents).toHaveBeenLastCalledWith(true);
vi.advanceTimersByTime(1000);
@@ -404,23 +404,33 @@ it("lets the viewer's pointer pass through, handing it back when it leaves or re
await settle();
// Centred pill: x 600–839, y 37–72.
const window = state.windows[0];
- overlay.viewerPointer({ x: 700, y: 50 });
- overlay.viewerPointer({ x: 710, y: 50 });
+ overlay.viewerPointer([{ x: 700, y: 50 }]);
+ overlay.viewerPointer([{ x: 710, y: 50 }]);
expect(window.setIgnoreMouseEvents.mock.calls).toEqual([[true]]);
- overlay.viewerPointer({ x: 700, y: 200 });
+ overlay.viewerPointer([{ x: 700, y: 200 }]);
expect(window.setIgnoreMouseEvents).toHaveBeenLastCalledWith(false);
// Resting over it returns it to local clicks; the viewer's next click re-arms it first.
- overlay.viewerPointer({ x: 700, y: 50 });
+ overlay.viewerPointer([{ x: 700, y: 50 }]);
vi.advanceTimersByTime(999);
expect(window.setIgnoreMouseEvents).toHaveBeenLastCalledWith(true);
vi.advanceTimersByTime(1);
expect(window.setIgnoreMouseEvents).toHaveBeenLastCalledWith(false);
- overlay.viewerPointer({ x: 700, y: 50 });
+ overlay.viewerPointer([{ x: 700, y: 50 }]);
expect(window.setIgnoreMouseEvents).toHaveBeenLastCalledWith(true);
+ // A click over it followed by a move away in the same batch still passes through.
+ overlay.viewerPointer([
+ { x: 700, y: 50 },
+ { x: 700, y: 200 },
+ ]);
+ expect(window.setIgnoreMouseEvents).toHaveBeenLastCalledWith(true);
+ // A later batch entirely clear of it hands it back.
+ overlay.viewerPointer([{ x: 700, y: 200 }]);
+ expect(window.setIgnoreMouseEvents).toHaveBeenLastCalledWith(false);
+ overlay.viewerPointer([{ x: 700, y: 50 }]);
// A lease that ends mid-rest leaves nothing behind for the next overlay.
overlay.update(null);
vi.runAllTimers();
- expect(window.setIgnoreMouseEvents).toHaveBeenCalledTimes(5);
+ expect(window.setIgnoreMouseEvents).toHaveBeenCalledTimes(7);
} finally {
vi.useRealTimers();
}
diff --git a/apps/desktop/src/main/remote-desktop/__tests__/inputHost.test.ts b/apps/desktop/src/main/remote-desktop/__tests__/inputHost.test.ts
index 397291e3af..f1cd8f67d1 100644
--- a/apps/desktop/src/main/remote-desktop/__tests__/inputHost.test.ts
+++ b/apps/desktop/src/main/remote-desktop/__tests__/inputHost.test.ts
@@ -354,8 +354,9 @@ describe('native input lifecycle', () => {
expect(c.child.stdin.write).not.toHaveBeenCalled();
c.exit();
});
- // The controlled overlay needs each viewer pointer before it lands to let it pass through.
- it('reports each whole-display pointer position before injecting it', async () => {
+ // The controlled overlay passes the viewer's pointer through only while its batch is
+ // injected: a batch queued behind an unacknowledged one must not change it early.
+ it('reports a batch’s pointer positions when its injection starts, not when queued', async () => {
const c = childProcess();
const pointer = vi.fn();
const host = new DesktopInputHost(
@@ -368,20 +369,34 @@ describe('native input lifecycle', () => {
return c.typed;
},
},
- (point) => pointer(point, c.child.stdin.write.mock.calls.length),
+ (points) => pointer(points, c.child.stdin.write.mock.calls.length),
);
await host.start('1');
host.input([
- { kind: 'move', x: 0.25, y: 0.75 },
- { kind: 'key', code: 'KeyA', down: true },
{ kind: 'button', button: 0, down: true, x: 0.5, y: 0.5 },
+ { kind: 'key', code: 'KeyA', down: true },
+ { kind: 'move', x: 0.25, y: 0.75 },
]);
+ host.input([{ kind: 'move', x: 0.1, y: 0.1 }]);
+ host.input([{ kind: 'key', code: 'KeyB', down: true }]);
await flush();
+ // Only the first batch has started; it reports every position at once, before its write.
expect(pointer.mock.calls).toEqual([
- [{ x: 25, y: 74 }, 0],
- [{ x: 50, y: 50 }, 0],
+ [
+ [
+ { x: 50, y: 50 },
+ { x: 25, y: 74 },
+ ],
+ 0,
+ ],
]);
- expect(c.child.stdin.write).toHaveBeenCalledOnce();
+ c.child.stdout.emit('data', Buffer.from('ok\n'));
+ await flush();
+ expect(pointer.mock.calls.at(-1)).toEqual([[{ x: 10, y: 10 }], 1]);
+ c.child.stdout.emit('data', Buffer.from('ok\n'));
+ await flush();
+ // A batch without pointer events never reports.
+ expect(pointer).toHaveBeenCalledTimes(2);
host.stop();
c.exit();
});
diff --git a/apps/desktop/src/main/remote-desktop/controlledOverlay.ts b/apps/desktop/src/main/remote-desktop/controlledOverlay.ts
index ec4b8330bc..1213446c76 100644
--- a/apps/desktop/src/main/remote-desktop/controlledOverlay.ts
+++ b/apps/desktop/src/main/remote-desktop/controlledOverlay.ts
@@ -194,17 +194,20 @@ export class ControlledOverlay {
}
}
- /** The viewer's pointer, in global DIP, just before it is injected. While it is
- * over the overlay or its confirmation, clicks pass through both; local clicks (on
- * the same system cursor) get them back once the viewer's pointer leaves or rests.
- * Each viewer button event re-arms pass-through before it is injected. */
- viewerPointer(point: { x: number; y: number }): void {
+ /** The viewer's pointer positions of one batch, in global DIP, reported as that
+ * batch starts being injected (every earlier batch has finished). If any is over the
+ * overlay or its confirmation, clicks pass through both for the whole batch; local
+ * clicks (on the same system cursor) get them back once a batch stays clear of
+ * them or the viewer's pointer rests. */
+ viewerPointer(points: { x: number; y: number }[]): void {
const windows = this.localOnlyWindows();
if (!windows.length) return;
- const over = windows.some((window) => {
- const { x, y, width, height } = window.getBounds();
- return point.x >= x && point.x < x + width && point.y >= y && point.y < y + height;
- });
+ const over = points.some((point) =>
+ windows.some((window) => {
+ const { x, y, width, height } = window.getBounds();
+ return point.x >= x && point.x < x + width && point.y >= y && point.y < y + height;
+ }),
+ );
if (this.restTimer) clearTimeout(this.restTimer);
this.restTimer = over
? setTimeout(() => this.setPassThrough(false), VIEWER_POINTER_REST_MS)
diff --git a/apps/desktop/src/main/remote-desktop/index.ts b/apps/desktop/src/main/remote-desktop/index.ts
index 19312aa409..352327644f 100644
--- a/apps/desktop/src/main/remote-desktop/index.ts
+++ b/apps/desktop/src/main/remote-desktop/index.ts
@@ -278,7 +278,7 @@ const input = new DesktopInputHost(
void remoteDesktop.releaseControl(true);
},
undefined,
- (point) => controlledOverlay.viewerPointer(point),
+ (points) => controlledOverlay.viewerPointer(points),
);
const clipboardCounter = new ClipboardCounter(resolveDesktopInputBinary);
// A display change holds native capture instead of stopping it: frames pause
diff --git a/apps/desktop/src/main/remote-desktop/inputHost.ts b/apps/desktop/src/main/remote-desktop/inputHost.ts
index 528a9b5438..3b99dd0f58 100644
--- a/apps/desktop/src/main/remote-desktop/inputHost.ts
+++ b/apps/desktop/src/main/remote-desktop/inputHost.ts
@@ -227,8 +227,9 @@ export class DesktopInputHost {
spawn: (binary: string, args: string[] = []) =>
spawn(binary, args, { stdio: 'pipe', windowsHide: true }),
},
- /** Each whole-display pointer position (global DIP), reported just before it is injected. */
- private readonly onPointer?: (point: { x: number; y: number }) => void,
+ /** The whole-display pointer positions (global DIP) of one batch, reported as that
+ * batch starts its native injection, i.e. after every earlier batch has finished. */
+ private readonly onPointer?: (points: { x: number; y: number }[]) => void,
) {}
async start(displayId: string, window?: { windowId: number; pid: number }): Promise<void> {
this.stop();
@@ -381,6 +382,7 @@ export class DesktopInputHost {
if (!display || (!this.child && !this.windows)) throw new Error('DESKTOP_INPUT_UNAVAILABLE');
if (events.length === 0) return;
const batch = this.activity!.begin(events);
+ const points: { x: number; y: number }[] = [];
this.write(
events.map((event) => {
if (event.kind !== 'move' && event.kind !== 'button') return event;
@@ -388,14 +390,19 @@ export class DesktopInputHost {
x: Math.round(display.bounds.x + event.x * (display.bounds.width - 1)),
y: Math.round(display.bounds.y + event.y * (display.bounds.height - 1)),
};
- this.onPointer?.(point);
+ points.push(point);
const native = process.platform === 'win32' ? screen.dipToScreenPoint(point) : point;
return { ...event, ...native };
}),
batch,
+ points,
);
}
- private write(events: unknown[], batch?: { ready: Promise<void>; complete: () => void }): void {
+ private write(
+ events: unknown[],
+ batch?: { ready: Promise<void>; complete: () => void },
+ points: { x: number; y: number }[] = [],
+ ): void {
const child = this.child;
const line = `${JSON.stringify(events)}\n`;
const connection = this.windows;
@@ -410,6 +417,9 @@ export class DesktopInputHost {
if (generation !== this.generation) return;
this.writing = this.writing.then(async () => {
if (generation !== this.generation) return;
+ // Local-only windows must pass this batch's pointer through exactly while it is
+ // injected, not when it was queued behind earlier batches.
+ if (points.length) this.onPointer?.(points);
if (connection) {
if ((await connection.request(line.slice(0, -1))) !== 'ok\n')
throw new Error('DESKTOP_INPUT_UNAVAILABLE');