A team of long-lived bots for DeepSeek Harness, with main bots, bot-to-bot messages and group chats.
Fix the cursor's geometry lookup and window-at-point checker
feiz2c482f6ab9d6
diff --git a/app/src/computer-cursor.mjs b/app/src/computer-cursor.mjsindex 13e303ef..79f5a53a 100644--- a/app/src/computer-cursor.mjs+++ b/app/src/computer-cursor.mjs@@ -161,7 +161,6 @@ export function createPointChecker({ platform, ownPid = process.pid, spawn = reaconst JXA_CHECKER = `ObjC.import('Foundation')ObjC.import('CoreGraphics')-var own = Number($.NSProcessInfo.processInfo.environment.objectForKey('DSBOT_OWN_PID').js || 0)var input = $.NSFileHandle.fileHandleWithStandardInputvar output = $.NSFileHandle.fileHandleWithStandardOutputvar buf = ''@@ -174,27 +173,28 @@ while (true) {if (!data || data.length === 0) breakbuf += $.NSString.alloc.initWithDataEncoding(data, $.NSUTF8StringEncoding).jsvar nl- while ((nl = buf.indexOf('\\n')) >= 0) {+ while ((nl = buf.indexOf('\n')) >= 0) {var line = buf.slice(0, nl); buf = buf.slice(nl + 1)- var parts = line.trim().split(/\\s+/)+ var parts = line.trim().split(/\s+/)var x = Number(parts[0]), y = Number(parts[1]), pid = Number(parts[2])var yes = falsetry {- var list = $.CGWindowListCopyWindowInfo($.kCGWindowListOptionOnScreenOnly, 0)+ var ref = $.CGWindowListCopyWindowInfo(1, 0)+ var list = ObjC.castRefToObject(ref)var count = list ? list.count : 0for (var i = 0; i < count && !yes; i++) {var win = list.objectAtIndex(i)- var owner = Number(win.objectForKey('kCGWindowOwnerPID') || 0)- if (owner === own) continue- if (Number(win.objectForKey('kCGWindowLayer') || 0) !== 0) continue+ var owner = ObjC.unwrap(win.objectForKey('kCGWindowOwnerPID'))+ var layer = ObjC.unwrap(win.objectForKey('kCGWindowLayer'))+ if (Number(layer) !== 0) continuevar b = win.objectForKey('kCGWindowBounds')if (!b) continue- var bx = Number(b.objectForKey('X') || 0), by = Number(b.objectForKey('Y') || 0)- var bw = Number(b.objectForKey('Width') || 0), bh = Number(b.objectForKey('Height') || 0)- if (x >= bx && x < bx + bw && y >= by && y < by + bh) { yes = (owner === pid); break }+ var bx = Number(ObjC.unwrap(b.objectForKey('X'))), by = Number(ObjC.unwrap(b.objectForKey('Y')))+ var bw = Number(ObjC.unwrap(b.objectForKey('Width'))), bh = Number(ObjC.unwrap(b.objectForKey('Height')))+ if (x >= bx && x < bx + bw && y >= by && y < by + bh) { yes = (Number(owner) === pid); break }}} catch (e) {}- answer(yes ? 'y\\n' : 'n\\n')+ answer(yes ? 'y\n' : 'n\n')}}`@@ -206,20 +206,24 @@ Add-Type @"using System;using System.Runtime.InteropServices;public class PointCheck {- [DllImport("user32.dll")] public static extern IntPtr WindowFromPoint(System.Drawing.Point p);+ [StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; public int Y; }+ [DllImport("user32.dll")] public static extern IntPtr WindowFromPoint(POINT p);[DllImport("user32.dll")] public static extern IntPtr GetAncestor(IntPtr hwnd, uint flags);[DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hwnd, out uint pid);+ [DllImport("user32.dll")] public static extern bool SetProcessDPIAware();}"@-$own = [int]$env:DSBOT_OWN_PID+[void][PointCheck]::SetProcessDPIAware()+# WindowFromPoint already skips our click-through (WS_EX_TRANSPARENT) overlays.while (($line = [Console]::In.ReadLine()) -ne $null) {$parts = $line.Trim() -split '\s+'if ($parts.Count -lt 3) { continue }- $p = New-Object System.Drawing.Point([int]$parts[0], [int]$parts[1])+ $p = New-Object PointCheck+POINT+ $p.X = [int]$parts[0]; $p.Y = [int]$parts[1]$h = [PointCheck]::WindowFromPoint($p)$root = [PointCheck]::GetAncestor($h, 2)[uint32]$pid2 = 0[void][PointCheck]::GetWindowThreadProcessId($root, [ref]$pid2)- if ($pid2 -eq $own) { [Console]::Out.WriteLine('y') } elseif ($pid2 -eq [uint32]$parts[2]) { [Console]::Out.WriteLine('y') } else { [Console]::Out.WriteLine('n') }+ if ($pid2 -eq [uint32]$parts[2]) { [Console]::Out.WriteLine('y') } else { [Console]::Out.WriteLine('n') }}`diff --git a/ds-bot/client.js b/ds-bot/client.jsindex 593f6ad1..b19d5b68 100644--- a/ds-bot/client.js+++ b/ds-bot/client.js@@ -25337,6 +25337,7 @@ function apply(ctx) {});for (const event of live2.cursors ?? []) {if ((seenCursors.get(event.botId) ?? 0) >= event.seq) continue;+ if (Number.isFinite(event.t) && Date.now() - event.t > 3e3) continue;seenCursors.set(event.botId, event.seq);if (typeof appComputer.computerCursor === "function") {appComputer.computerCursor({ ...event, color: look(event.botId).color, avatar: look(event.botId).image });diff --git a/ds-bot/src/host/computer/index.js b/ds-bot/src/host/computer/index.jsindex 84efba95..a0d5c677 100644--- a/ds-bot/src/host/computer/index.js+++ b/ds-bot/src/host/computer/index.js@@ -429,6 +429,7 @@ export function install(rt) {return map}const cursors = new Map()+ const lastWindowOf = new Map()let cursorSeq = 0const publishCursor = (botId, event) => {cursors.set(botId, { botId, seq: ++cursorSeq, t: Date.now(), ...event })@@ -437,6 +438,10 @@ export function install(rt) {const RIPPLE_METHODS = new Set(['click', 'double_click', 'performSecondaryAction', 'drag'])const recordGeo = (sessionId, result) => {if (sessionId === undefined || result === null || typeof result !== 'object') return+ if (typeof result.appHandle === 'string') {+ const windowId = result.windowId ?? result.focusedWindowId ?? (Array.isArray(result.windows) ? result.windows[0]?.windowId : undefined)+ if (windowId !== undefined) lastWindowOf.set(`${sessionId}|${result.appHandle}`, windowId)+ }const windows = Array.isArray(result.windows) ? result.windows : []for (const win of windows) {const windowId = win?.windowId ?? win?.id@@ -468,7 +473,8 @@ export function install(rt) {if (botId === undefined || !rt.isBotSession(sessionId)) returnif (call.provider === 'cua') {const params = exec.arguments?.params ?? {}- const windowId = params.windowId ?? result?.focusedWindowId ?? result?.windowId+ const remembered = typeof params.appHandle === 'string' ? lastWindowOf.get(`${sessionId}|${params.appHandle}`) : undefined+ const windowId = params.windowId ?? remembered ?? result?.focusedWindowId ?? result?.windowIdconst win = windowId === undefined ? undefined : geoMapOf(sessionId).get(windowId)if (win === undefined || !Number.isFinite(win.x) || !Number.isFinite(win.y) || !Number.isFinite(win.w) || !Number.isFinite(win.h)) returnconst common = { pid: win.pid, windowId }