Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-bench): raise when the sandbox does not answer in file_exists/directory_exists (#5194)
Tanish Gudise1ff57101e705
diff --git a/libs/cua-bench/cua_bench/compat/legacy_interface.py b/libs/cua-bench/cua_bench/compat/legacy_interface.pyindex fe8cc210d8..893a0a25de 100644--- a/libs/cua-bench/cua_bench/compat/legacy_interface.py+++ b/libs/cua-bench/cua_bench/compat/legacy_interface.py@@ -24,6 +24,7 @@ from __future__ import annotationsimport asyncioimport base64+import functoolsimport jsonimport loggingimport shlex@@ -57,6 +58,36 @@ def _unb64(text: str) -> bytes:return base64.b64decode(text or "")[email protected]_cache(maxsize=1)+def _unreachable_errors() -> Tuple[type, ...]:+ """Error types that mean the sandbox did not answer (what is installed of them)."""+ errors: List[type] = [TimeoutError, ConnectionError]+ try:+ from cua_sandbox import SpacesdNotAvailable++ errors.append(SpacesdNotAvailable)+ except Exception: # noqa: BLE001 - cua-sandbox not importable here+ pass+ try:+ from cua import CuaError+ except Exception: # noqa: BLE001 - no native SDK in this environment+ CuaError = None+ for name in ("Transport", "Timeout", "Closed", "SpacesdNotAvailable", "DaemonNotRunning"):+ cls = getattr(CuaError, name, None)+ if isinstance(cls, type):+ errors.append(cls)+ return tuple(errors)+++def _unreachable(error: BaseException) -> bool:+ """True when ``error`` means the sandbox could not be reached or timed out.++ cua-sandbox already answers a path that does not exist with False rather+ than an error, so these are never a "not found".+ """+ return isinstance(error, _unreachable_errors())++class LegacyInterface:"""cua-computer's ``BaseComputerInterface`` methods over ``_send_command``.@@ -435,17 +466,27 @@ class SandboxInterface(LegacyInterface):# direct paths (no base64 round trip) -------------------------------------async def file_exists(self, path: str) -> bool:+ """Whether ``path`` is an existing file.++ A sandbox that could not be reached raises: that is not an answer, and+ a task scoring "missing" from it would record a reward of 0.+ """files = self._sb.filestry:return bool(await files.exists(path)) and not bool(await files.is_dir(path))- except Exception: # noqa: BLE001+ except Exception as error: # noqa: BLE001+ if _unreachable(error):+ raisereturn Falseasync def directory_exists(self, path: str) -> bool:+ """Whether ``path`` is an existing directory; raises like :meth:`file_exists`."""files = self._sb.filestry:return bool(await files.exists(path)) and bool(await files.is_dir(path))- except Exception: # noqa: BLE001+ except Exception as error: # noqa: BLE001+ if _unreachable(error):+ raisereturn Falseasync def read_bytes(self, path: str, offset: int = 0, length: Optional[int] = None) -> bytes:diff --git a/libs/cua-bench/cua_bench/computers/remote.py b/libs/cua-bench/cua_bench/computers/remote.pyindex 862129ed74..500bd627fa 100644--- a/libs/cua-bench/cua_bench/computers/remote.py+++ b/libs/cua-bench/cua_bench/computers/remote.py@@ -1291,12 +1291,18 @@ class RemoteDesktopSession:await self.interface.write_bytes(path, data)async def file_exists(self, path: str) -> bool:- """Whether ``path`` is an existing file (not a directory), as in 0.2.x."""+ """Whether ``path`` is an existing file (not a directory), as in 0.2.x.++ On a cua-sandbox session, raises if the sandbox does not answer.+ """await self._ensure_computer()return await self.interface.file_exists(path)async def directory_exists(self, path: str) -> bool:- """Check if a directory exists in the environment."""+ """Check if a directory exists in the environment.++ On a cua-sandbox session, raises if the sandbox does not answer.+ """await self._ensure_computer()return await self.interface.directory_exists(path)diff --git a/libs/cua-bench/cua_bench/tests/test_sandbox_exists_errors.py b/libs/cua-bench/cua_bench/tests/test_sandbox_exists_errors.pynew file mode 100644index 0000000000..35eb145aca--- /dev/null+++ b/libs/cua-bench/cua_bench/tests/test_sandbox_exists_errors.py@@ -0,0 +1,88 @@+"""file_exists / directory_exists on a sandbox session when the sandbox does not answer.++A task's ``evaluate`` commonly scores "answer file missing" as 0 (the starter+template and example_tasks/hello_file_env do). If a sandbox that stopped+answering also reads as "missing", a frozen or crashed sandbox records a+reward of 0 for an episode nobody scored. An unreachable sandbox has to raise.+"""++from __future__ import annotations++import pytest+from cua_bench.computers.remote import RemoteDesktopSession++from .fakes import FakeSandbox+++class _FailingFiles:+ """The files API of a sandbox whose every call fails with ``error``."""++ def __init__(self, error: BaseException) -> None:+ self.error = error++ async def exists(self, path: str) -> bool:+ raise self.error++ async def is_dir(self, path: str) -> bool:+ raise self.error+++def _session(files=None) -> RemoteDesktopSession:+ sb = FakeSandbox()+ if files is not None:+ sb.files = files+ return RemoteDesktopSession.attach(sb, os_type="linux")+++def _spacesd_not_available():+ from cua_sandbox import SpacesdNotAvailable++ return SpacesdNotAvailable("no cua-spacesd answered")+++ "error",+ [+ TimeoutError("operation timed out"),+ ConnectionResetError("connection reset by peer"),+ _spacesd_not_available(),+ ],+ ids=["timeout", "connection", "spacesd-not-available"],+)+async def test_an_unreachable_sandbox_raises_instead_of_reading_as_missing(error):+ session = _session(_FailingFiles(error))+ with pytest.raises(type(error)):+ await session.file_exists("/tmp/cb_answer.txt")+ with pytest.raises(type(error)):+ await session.directory_exists("/tmp")+++async def test_the_native_transport_error_raises_and_not_found_reads_as_missing():+ try:+ from cua import CuaError+ except Exception: # noqa: BLE001 - the native SDK library is not built here+ pytest.skip("native cua SDK not available")++ frozen = CuaError.Transport("transport error: http2 error (operation timed out)")+ with pytest.raises(type(frozen)):+ await _session(_FailingFiles(frozen)).file_exists("/tmp/cb_answer.txt")+ with pytest.raises(CuaError.Timeout):+ await _session(_FailingFiles(CuaError.Timeout("deadline exceeded"))).directory_exists("/t")+ missing = _session(_FailingFiles(CuaError.NotFound("no such file")))+ assert await missing.file_exists("/tmp/cb_answer.txt") is False+++async def test_present_and_missing_paths_answer_as_before():+ session = _session()+ await session.write_file("/tmp/cb_answer.txt", "hello")+ assert await session.file_exists("/tmp/cb_answer.txt") is True+ assert await session.file_exists("/tmp/missing.txt") is False+ assert await session.file_exists("/tmp") is False+ assert await session.directory_exists("/tmp") is True+ assert await session.directory_exists("/tmp/missing") is False+++async def test_other_errors_still_read_as_missing():+ session = _session(_FailingFiles(PermissionError("permission denied")))+ assert await session.file_exists("/root/secret.txt") is False+ assert await session.directory_exists("/root") is False