Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): emit structuredContent on get_window_state

Francesco Bonacci17b1a5e25a30
diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/GetWindowStateTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/GetWindowStateTool.swift
index 018d43ebc2..18e0ee4d29 100644
--- a/libs/cua-driver/Sources/CuaDriverServer/Tools/GetWindowStateTool.swift
+++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/GetWindowStateTool.swift
@@ -261,6 +261,29 @@ public enum GetWindowStateTool {
)
}
content.append(.text(text: textContent, annotations: nil, _meta: nil))
+ // Strip the b64 bytes from the structured snapshot — the image
+ // is already the first content block and tests just need the
+ // metadata fields (screenshot_scale_factor, dimensions, etc.).
+ let structuredSnapshot = AppStateSnapshot(
+ pid: snapshot.pid,
+ bundleId: snapshot.bundleId,
+ name: snapshot.name,
+ treeMarkdown: snapshot.treeMarkdown,
+ elementCount: snapshot.elementCount,
+ turnId: snapshot.turnId,
+ screenshotPngBase64: nil,
+ screenshotWidth: snapshot.screenshotWidth,
+ screenshotHeight: snapshot.screenshotHeight,
+ screenshotScaleFactor: snapshot.screenshotScaleFactor,
+ screenshotOriginalWidth: snapshot.screenshotOriginalWidth,
+ screenshotOriginalHeight: snapshot.screenshotOriginalHeight
+ )
+ if let result = try? CallTool.Result(
+ content: content,
+ structuredContent: structuredSnapshot
+ ) {
+ return result
+ }
return CallTool.Result(content: content)
} catch let error as AppStateError {
return errorResult(error.description)