Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): emit structuredContent on get_window_state
Francesco Bonacci17b1a5e25a30
diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/GetWindowStateTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/GetWindowStateTool.swiftindex 018d43ebc2..18e0ee4d29 100644--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/GetWindowStateTool.swift+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/GetWindowStateTool.swift@@ -261,6 +261,29 @@ public enum GetWindowStateTool {)}content.append(.text(text: textContent, annotations: nil, _meta: nil))+ // Strip the b64 bytes from the structured snapshot — the image+ // is already the first content block and tests just need the+ // metadata fields (screenshot_scale_factor, dimensions, etc.).+ let structuredSnapshot = AppStateSnapshot(+ pid: snapshot.pid,+ bundleId: snapshot.bundleId,+ name: snapshot.name,+ treeMarkdown: snapshot.treeMarkdown,+ elementCount: snapshot.elementCount,+ turnId: snapshot.turnId,+ screenshotPngBase64: nil,+ screenshotWidth: snapshot.screenshotWidth,+ screenshotHeight: snapshot.screenshotHeight,+ screenshotScaleFactor: snapshot.screenshotScaleFactor,+ screenshotOriginalWidth: snapshot.screenshotOriginalWidth,+ screenshotOriginalHeight: snapshot.screenshotOriginalHeight+ )+ if let result = try? CallTool.Result(+ content: content,+ structuredContent: structuredSnapshot+ ) {+ return result+ }return CallTool.Result(content: content)} catch let error as AppStateError {return errorResult(error.description)