Skip to content

A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.

Fix the cursor's geometry lookup and window-at-point checker

feiz2c482f6ab9d6
diff --git a/app/src/computer-cursor.mjs b/app/src/computer-cursor.mjs
index 13e303ef..79f5a53a 100644
--- a/app/src/computer-cursor.mjs
+++ b/app/src/computer-cursor.mjs
@@ -161,7 +161,6 @@ export function createPointChecker({ platform, ownPid = process.pid, spawn = rea
const JXA_CHECKER = `
ObjC.import('Foundation')
ObjC.import('CoreGraphics')
-var own = Number($.NSProcessInfo.processInfo.environment.objectForKey('DSBOT_OWN_PID').js || 0)
var input = $.NSFileHandle.fileHandleWithStandardInput
var output = $.NSFileHandle.fileHandleWithStandardOutput
var buf = ''
@@ -174,27 +173,28 @@ while (true) {
if (!data || data.length === 0) break
buf += $.NSString.alloc.initWithDataEncoding(data, $.NSUTF8StringEncoding).js
var nl
- while ((nl = buf.indexOf('\\n')) >= 0) {
+ while ((nl = buf.indexOf('\n')) >= 0) {
var line = buf.slice(0, nl); buf = buf.slice(nl + 1)
- var parts = line.trim().split(/\\s+/)
+ var parts = line.trim().split(/\s+/)
var x = Number(parts[0]), y = Number(parts[1]), pid = Number(parts[2])
var yes = false
try {
- var list = $.CGWindowListCopyWindowInfo($.kCGWindowListOptionOnScreenOnly, 0)
+ var ref = $.CGWindowListCopyWindowInfo(1, 0)
+ var list = ObjC.castRefToObject(ref)
var count = list ? list.count : 0
for (var i = 0; i < count && !yes; i++) {
var win = list.objectAtIndex(i)
- var owner = Number(win.objectForKey('kCGWindowOwnerPID') || 0)
- if (owner === own) continue
- if (Number(win.objectForKey('kCGWindowLayer') || 0) !== 0) continue
+ var owner = ObjC.unwrap(win.objectForKey('kCGWindowOwnerPID'))
+ var layer = ObjC.unwrap(win.objectForKey('kCGWindowLayer'))
+ if (Number(layer) !== 0) continue
var b = win.objectForKey('kCGWindowBounds')
if (!b) continue
- var bx = Number(b.objectForKey('X') || 0), by = Number(b.objectForKey('Y') || 0)
- var bw = Number(b.objectForKey('Width') || 0), bh = Number(b.objectForKey('Height') || 0)
- if (x >= bx && x < bx + bw && y >= by && y < by + bh) { yes = (owner === pid); break }
+ var bx = Number(ObjC.unwrap(b.objectForKey('X'))), by = Number(ObjC.unwrap(b.objectForKey('Y')))
+ var bw = Number(ObjC.unwrap(b.objectForKey('Width'))), bh = Number(ObjC.unwrap(b.objectForKey('Height')))
+ if (x >= bx && x < bx + bw && y >= by && y < by + bh) { yes = (Number(owner) === pid); break }
}
} catch (e) {}
- answer(yes ? 'y\\n' : 'n\\n')
+ answer(yes ? 'y\n' : 'n\n')
}
}
`
@@ -206,20 +206,24 @@ Add-Type @"
using System;
using System.Runtime.InteropServices;
public class PointCheck {
- [DllImport("user32.dll")] public static extern IntPtr WindowFromPoint(System.Drawing.Point p);
+ [StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; public int Y; }
+ [DllImport("user32.dll")] public static extern IntPtr WindowFromPoint(POINT p);
[DllImport("user32.dll")] public static extern IntPtr GetAncestor(IntPtr hwnd, uint flags);
[DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hwnd, out uint pid);
+ [DllImport("user32.dll")] public static extern bool SetProcessDPIAware();
}
"@
-$own = [int]$env:DSBOT_OWN_PID
+[void][PointCheck]::SetProcessDPIAware()
+# WindowFromPoint already skips our click-through (WS_EX_TRANSPARENT) overlays.
while (($line = [Console]::In.ReadLine()) -ne $null) {
$parts = $line.Trim() -split '\s+'
if ($parts.Count -lt 3) { continue }
- $p = New-Object System.Drawing.Point([int]$parts[0], [int]$parts[1])
+ $p = New-Object PointCheck+POINT
+ $p.X = [int]$parts[0]; $p.Y = [int]$parts[1]
$h = [PointCheck]::WindowFromPoint($p)
$root = [PointCheck]::GetAncestor($h, 2)
[uint32]$pid2 = 0
[void][PointCheck]::GetWindowThreadProcessId($root, [ref]$pid2)
- if ($pid2 -eq $own) { [Console]::Out.WriteLine('y') } elseif ($pid2 -eq [uint32]$parts[2]) { [Console]::Out.WriteLine('y') } else { [Console]::Out.WriteLine('n') }
+ if ($pid2 -eq [uint32]$parts[2]) { [Console]::Out.WriteLine('y') } else { [Console]::Out.WriteLine('n') }
}
`
diff --git a/ds-bot/client.js b/ds-bot/client.js
index 593f6ad1..b19d5b68 100644
--- a/ds-bot/client.js
+++ b/ds-bot/client.js
@@ -25337,6 +25337,7 @@ function apply(ctx) {
});
for (const event of live2.cursors ?? []) {
if ((seenCursors.get(event.botId) ?? 0) >= event.seq) continue;
+ if (Number.isFinite(event.t) && Date.now() - event.t > 3e3) continue;
seenCursors.set(event.botId, event.seq);
if (typeof appComputer.computerCursor === "function") {
appComputer.computerCursor({ ...event, color: look(event.botId).color, avatar: look(event.botId).image });
diff --git a/ds-bot/src/host/computer/index.js b/ds-bot/src/host/computer/index.js
index 84efba95..a0d5c677 100644
--- a/ds-bot/src/host/computer/index.js
+++ b/ds-bot/src/host/computer/index.js
@@ -429,6 +429,7 @@ export function install(rt) {
return map
}
const cursors = new Map()
+ const lastWindowOf = new Map()
let cursorSeq = 0
const publishCursor = (botId, event) => {
cursors.set(botId, { botId, seq: ++cursorSeq, t: Date.now(), ...event })
@@ -437,6 +438,10 @@ export function install(rt) {
const RIPPLE_METHODS = new Set(['click', 'double_click', 'performSecondaryAction', 'drag'])
const recordGeo = (sessionId, result) => {
if (sessionId === undefined || result === null || typeof result !== 'object') return
+ if (typeof result.appHandle === 'string') {
+ const windowId = result.windowId ?? result.focusedWindowId ?? (Array.isArray(result.windows) ? result.windows[0]?.windowId : undefined)
+ if (windowId !== undefined) lastWindowOf.set(`${sessionId}|${result.appHandle}`, windowId)
+ }
const windows = Array.isArray(result.windows) ? result.windows : []
for (const win of windows) {
const windowId = win?.windowId ?? win?.id
@@ -468,7 +473,8 @@ export function install(rt) {
if (botId === undefined || !rt.isBotSession(sessionId)) return
if (call.provider === 'cua') {
const params = exec.arguments?.params ?? {}
- const windowId = params.windowId ?? result?.focusedWindowId ?? result?.windowId
+ const remembered = typeof params.appHandle === 'string' ? lastWindowOf.get(`${sessionId}|${params.appHandle}`) : undefined
+ const windowId = params.windowId ?? remembered ?? result?.focusedWindowId ?? result?.windowId
const win = windowId === undefined ? undefined : geoMapOf(sessionId).get(windowId)
if (win === undefined || !Number.isFinite(win.x) || !Number.isFinite(win.y) || !Number.isFinite(win.w) || !Number.isFinite(win.h)) return
const common = { pid: win.pid, windowId }