Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
lume: avoid black VNC windows at VM startup (#1132)
Francesco Bonacci31a512de6903
diff --git a/libs/lume/src/VM/VM.swift b/libs/lume/src/VM/VM.swiftindex 014ebf483a..8e2d2b9c2d 100644--- a/libs/lume/src/VM/VM.swift+++ b/libs/lume/src/VM/VM.swift@@ -1,4 +1,5 @@import Foundation+import CoreGraphics// MARK: - Support Types@@ -245,7 +246,7 @@ class VM {"port": "\(vncPort)",])let vncInfo = try await setupSession(- noDisplay: noDisplay, port: vncPort, sharedDirectories: sharedDirectories)+ port: vncPort, sharedDirectories: sharedDirectories)Logger.info("VNC setup successful", metadata: ["name": vmDirContext.name, "vncInfo": vncInfo])@@ -259,6 +260,13 @@ class VM {try await service.start()Logger.info("VM started successfully", metadata: ["name": vmDirContext.name])+ // Open the VNC client only after VM start to avoid connecting to an empty framebuffer.+ if !noDisplay {+ await waitForVisibleFramebufferBeforeOpeningClient()+ Logger.info("Starting VNC session", metadata: ["name": vmDirContext.name])+ try await vncService.openClient(url: vncInfo)+ }+// Start clipboard watcher for automatic host-to-VM clipboard sync// Requires SSH/Remote Login to be enabled on the VMif clipboard {@@ -684,7 +692,7 @@ class VM {/// Main session setup method that handles VNC and persists session dataprivate func setupSession(- noDisplay: Bool, port: Int = 0, sharedDirectories: [SharedDirectory] = []+ port: Int = 0, sharedDirectories: [SharedDirectory] = []) async throws -> String {// Start the VNC service and get the URLlet url = try await startVNCService(port: port)@@ -692,13 +700,71 @@ class VM {// Save the session datasaveSessionData(url: url, sharedDirectories: sharedDirectories)- // Open the VNC client if needed- if !noDisplay {- Logger.info("Starting VNC session", metadata: ["name": vmDirContext.name])- try await vncService.openClient(url: url)+ return url+ }++ /// Avoid opening Screen Sharing on an all-black initial framebuffer.+ /// This only runs for the real VNC service (not mocks in tests).+ private func waitForVisibleFramebufferBeforeOpeningClient() async {+ guard vncService is DefaultVNCService else {+ return}- return url+ do {+ try await vncService.connectInputClient()+ defer { vncService.disconnectInputClient() }++ let timeoutSeconds = 30+ for _ in 0..<timeoutSeconds {+ if let image = try? await vncService.captureFramebuffer(),+ framebufferHasVisiblePixels(image)+ {+ Logger.info(+ "Detected visible VM framebuffer content before opening VNC client",+ metadata: ["name": vmDirContext.name]+ )+ return+ }+ try? await Task.sleep(nanoseconds: 1_000_000_000)+ }++ Logger.info(+ "Timed out waiting for visible framebuffer content; opening VNC client anyway",+ metadata: ["name": vmDirContext.name, "timeout_seconds": "\(timeoutSeconds)"]+ )+ } catch {+ Logger.info(+ "Framebuffer readiness check failed; opening VNC client anyway",+ metadata: ["name": vmDirContext.name, "error": "\(error)"]+ )+ }+ }++ /// Fast heuristic: sample bytes from the framebuffer and treat any non-zero value as visible content.+ private func framebufferHasVisiblePixels(_ image: CGImage) -> Bool {+ guard let dataProvider = image.dataProvider,+ let data = dataProvider.data,+ let bytes = CFDataGetBytePtr(data)+ else {+ // If we can't inspect pixels, do not block client opening.+ return true+ }++ let count = CFDataGetLength(data)+ guard count > 0 else {+ return false+ }++ let stride = max(1, count / 4096)+ var index = 0+ while index < count {+ if bytes[index] != 0 {+ return true+ }+ index += stride+ }++ return false}// MARK: - Platform-specific Methods@@ -847,7 +913,7 @@ class VM {virtualizationService = try virtualizationServiceFactory(config)let vncInfo = try await setupSession(- noDisplay: noDisplay, port: vncPort, sharedDirectories: sharedDirectories)+ port: vncPort, sharedDirectories: sharedDirectories)Logger.info("VNC info", metadata: ["vncInfo": vncInfo])// Start the VM@@ -856,6 +922,12 @@ class VM {}try await service.start()+ if !noDisplay {+ await waitForVisibleFramebufferBeforeOpeningClient()+ Logger.info("Starting VNC session", metadata: ["name": vmDirContext.name])+ try await vncService.openClient(url: vncInfo)+ }+while true {try await Task.sleep(nanoseconds: UInt64(1e9))}diff --git a/libs/lume/src/VNC/VNCService.swift b/libs/lume/src/VNC/VNCService.swiftindex 8b7ab8854a..ed41c3b684 100644--- a/libs/lume/src/VNC/VNCService.swift+++ b/libs/lume/src/VNC/VNCService.swift@@ -184,7 +184,18 @@ final class DefaultVNCService: VNCService {func openClient(url: String) async throws {let processRunner = DefaultProcessRunner()- try processRunner.run(executable: "/usr/bin/open", arguments: [url])+ do {+ // Force a fresh Screen Sharing instance to avoid stale/paused sessions.+ try processRunner.run(+ executable: "/usr/bin/open",+ arguments: ["-n", "-a", "Screen Sharing", url]+ )+ } catch {+ Logger.info("Failed to open Screen Sharing explicitly, falling back to default URL open", metadata: [+ "error": "\(error)"+ ])+ try processRunner.run(executable: "/usr/bin/open", arguments: [url])+ }}/// Connect a VNC client to the server for sending input events@@ -419,4 +430,4 @@ final class DefaultVNCService: VNCService {Logger.debug("Char with modifiers sent via VNC client")}-}\ No newline at end of file+}