👾 Open Computer Use – Open-Source Alternative to Codex Computer Use
Swift accessibilityai-agentai-agentsclaude-codecodexcodex-computer-usecomputer-usecomputer-use-agentcross-platformcuadesktop-automationgemini-cligui-automationmacosmcpmodel-context-protocolopen-computer-useopen-computer-use-mcpopenai-codexopencode
fix(drag): drive real window-server drags (deltas, event number, HID posting)
Camille Clayton5c7699244c9f
diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/InputSimulation.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/InputSimulation.swiftindex fa66f84..5a06785 100644--- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/InputSimulation.swift+++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/InputSimulation.swift@@ -140,23 +140,29 @@ enum InputSimulation {}static func dragGlobally(from start: CGPoint, to end: CGPoint) throws {- guard let source = CGEventSource(stateID: .hidSystemState) else {- throw ComputerUseError.message("Failed to create HID event source.")- }-- try postMouseEvent(type: .mouseMoved, source: source, point: start, button: .left, clickState: 1)- try postMouseEvent(type: .leftMouseDown, source: source, point: start, button: .left, clickState: 1)-- for step in 1...10 {- let progress = CGFloat(step) / 10+ // A nil (default) event source posted to the HID tap, with per-step deltas and+ // no timestamp override. This is what lets macOS recognize the synthetic+ // sequence as a real window-server drag on current releases.+ let gesture = dragGestureEventNumber()+ try postMouseEvent(type: .mouseMoved, source: nil, point: start, button: .left, clickState: 1)+ try postMouseEvent(type: .leftMouseDown, source: nil, point: start, button: .left, clickState: 1, eventNumber: gesture)+ // Clear the OS drag threshold before motion, or the gesture reads as a click.+ Thread.sleep(forTimeInterval: 0.05)++ var previous = start+ let steps = dragStepCount(from: start, to: end)+ for step in 1...steps {+ let progress = CGFloat(step) / CGFloat(steps)let point = CGPoint(x: start.x + ((end.x - start.x) * progress),y: start.y + ((end.y - start.y) * progress))- try postMouseEvent(type: .leftMouseDragged, source: source, point: point, button: .left, clickState: 1)+ let delta = CGSize(width: point.x - previous.x, height: point.y - previous.y)+ try postMouseEvent(type: .leftMouseDragged, source: nil, point: point, button: .left, clickState: 1, delta: delta, eventNumber: gesture)+ previous = point}- try postMouseEvent(type: .leftMouseUp, source: source, point: end, button: .left, clickState: 1)+ try postMouseEvent(type: .leftMouseUp, source: nil, point: end, button: .left, clickState: 1, eventNumber: gesture)}static func typeText(_ text: String, pid: pid_t) throws {@@ -241,16 +247,59 @@ enum InputSimulation {Thread.sleep(forTimeInterval: 0.1)}- private static func postMouseEvent(type: CGEventType, source: CGEventSource, point: CGPoint, button: CGMouseButton, clickState: Int) throws {+ private static func postMouseEvent(type: CGEventType, source: CGEventSource?, point: CGPoint, button: CGMouseButton, clickState: Int, delta: CGSize? = nil, eventNumber: Int64? = nil) throws {guard let event = CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: point, mouseButton: button) else {throw ComputerUseError.message("Failed to create mouse event \(type.rawValue).")}event.setIntegerValueField(.mouseEventClickState, value: Int64(clickState))+ applyDragMotionFields(to: event, delta: delta, eventNumber: eventNumber)event.post(tap: .cghidEventTap)Thread.sleep(forTimeInterval: 0.03)}+ /// A synthetic macOS drag registers only when each dragged event carries the+ /// per-step movement delta: macOS reads kCGMouseEventDeltaX/Y (both the integer+ /// and double fields), not the absolute position, to decide the pointer moved.+ /// Applied only to drag events (delta != nil); clicks are unchanged.+ private static func applyDragMotionFields(to event: CGEvent, delta: CGSize?, eventNumber: Int64? = nil) {+ if eventNumber != nil || delta != nil {+ event.flags = []+ }+ if let eventNumber {+ event.setIntegerValueField(.mouseEventNumber, value: eventNumber)+ }+ guard let delta else { return }+ let dx = Int64(delta.width.rounded())+ let dy = Int64(delta.height.rounded())+ event.setIntegerValueField(.mouseEventDeltaX, value: dx)+ event.setIntegerValueField(.mouseEventDeltaY, value: dy)+ event.setDoubleValueField(.mouseEventDeltaX, value: Double(dx))+ event.setDoubleValueField(.mouseEventDeltaY, value: Double(dy))+ }++ /// Whether synthetic drag events need a matching gesture event number. Older+ /// macOS ignores it; recent releases require it (codex#43047 reports 26/27).+ static func needsDragEventNumber(majorVersion: Int) -> Bool { majorVersion >= 26 }++ /// A gesture identity shared by the down/dragged/up sequence, seeded above the+ /// system's last event number so recent macOS treats the sequence as one drag.+ private static func dragGestureEventNumber() -> Int64? {+ guard needsDragEventNumber(majorVersion: ProcessInfo.processInfo.operatingSystemVersion.majorVersion) else { return nil }+ let hid = CGEventSourceStateID.hidSystemState+ let base = Int64(CGEventSource.counterForEventType(hid, eventType: .leftMouseDown))+ + Int64(CGEventSource.counterForEventType(hid, eventType: .rightMouseDown))+ + Int64(CGEventSource.counterForEventType(hid, eventType: .otherMouseDown))+ return base + 1+ }++ /// Coarse interpolation makes large per-event jumps that macOS drag recognition+ /// can miss, so scale step count with distance (~4px/step) within a sane range.+ static func dragStepCount(from start: CGPoint, to end: CGPoint) -> Int {+ let distance = hypot(end.x - start.x, end.y - start.y)+ return min(max(Int((distance / 4).rounded(.up)), 10), 60)+ }+private static func postMouseEventToPid(type: CGEventType, source: CGEventSource, point: CGPoint, button: CGMouseButton, clickState: Int, pid: pid_t) throws {guard let event = CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: point, mouseButton: button) else {throw ComputerUseError.message("Failed to create mouse event \(type.rawValue).")diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swiftindex a83e625..4e7ea17 100644--- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift+++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift@@ -1475,6 +1475,22 @@ final class OpenComputerUseKitTests: XCTestCase {XCTAssertFalse(globalPointerFallbacksEnabled(environment: ["OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS": "false"]))}+ func testDragStepCountScalesWithDistanceAndClampsToBounds() {+ XCTAssertEqual(InputSimulation.dragStepCount(from: .zero, to: .zero), 10)+ XCTAssertEqual(InputSimulation.dragStepCount(from: CGPoint(x: 0, y: 0), to: CGPoint(x: 8, y: 0)), 10)+ XCTAssertEqual(InputSimulation.dragStepCount(from: CGPoint(x: 0, y: 0), to: CGPoint(x: 200, y: 0)), 50)+ XCTAssertEqual(InputSimulation.dragStepCount(from: CGPoint(x: 0, y: 0), to: CGPoint(x: 5000, y: 0)), 60)+ }++ func testNeedsDragEventNumberGatesOnRecentMacOS() {+ // Older macOS ignores the gesture event number; 26+ needs it (codex#43047).+ XCTAssertFalse(InputSimulation.needsDragEventNumber(majorVersion: 14))+ XCTAssertFalse(InputSimulation.needsDragEventNumber(majorVersion: 15))+ XCTAssertFalse(InputSimulation.needsDragEventNumber(majorVersion: 25))+ XCTAssertTrue(InputSimulation.needsDragEventNumber(majorVersion: 26))+ XCTAssertTrue(InputSimulation.needsDragEventNumber(majorVersion: 27))+ }+func testDragDeliveryPathFollowsGlobalPointerGate() {XCTAssertEqual(dragDeliveryPath(environment: [:]), .appPost)XCTAssertEqual(dragDeliveryPath(environment: ["OPEN_COMPUTER_USE_ALLOW_GLOBAL_POINTER_FALLBACKS": "0"]), .appPost)