Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-bench): raise when the sandbox does not answer in file_exists/directory_exists (#5194)

Tanish Gudise1ff57101e705
diff --git a/libs/cua-bench/cua_bench/compat/legacy_interface.py b/libs/cua-bench/cua_bench/compat/legacy_interface.py
index fe8cc210d8..893a0a25de 100644
--- a/libs/cua-bench/cua_bench/compat/legacy_interface.py
+++ b/libs/cua-bench/cua_bench/compat/legacy_interface.py
@@ -24,6 +24,7 @@ from __future__ import annotations
import asyncio
import base64
+import functools
import json
import logging
import shlex
@@ -57,6 +58,36 @@ def _unb64(text: str) -> bytes:
return base64.b64decode(text or "")
[email protected]_cache(maxsize=1)
+def _unreachable_errors() -> Tuple[type, ...]:
+ """Error types that mean the sandbox did not answer (what is installed of them)."""
+ errors: List[type] = [TimeoutError, ConnectionError]
+ try:
+ from cua_sandbox import SpacesdNotAvailable
+
+ errors.append(SpacesdNotAvailable)
+ except Exception: # noqa: BLE001 - cua-sandbox not importable here
+ pass
+ try:
+ from cua import CuaError
+ except Exception: # noqa: BLE001 - no native SDK in this environment
+ CuaError = None
+ for name in ("Transport", "Timeout", "Closed", "SpacesdNotAvailable", "DaemonNotRunning"):
+ cls = getattr(CuaError, name, None)
+ if isinstance(cls, type):
+ errors.append(cls)
+ return tuple(errors)
+
+
+def _unreachable(error: BaseException) -> bool:
+ """True when ``error`` means the sandbox could not be reached or timed out.
+
+ cua-sandbox already answers a path that does not exist with False rather
+ than an error, so these are never a "not found".
+ """
+ return isinstance(error, _unreachable_errors())
+
+
class LegacyInterface:
"""cua-computer's ``BaseComputerInterface`` methods over ``_send_command``.
@@ -435,17 +466,27 @@ class SandboxInterface(LegacyInterface):
# direct paths (no base64 round trip) -------------------------------------
async def file_exists(self, path: str) -> bool:
+ """Whether ``path`` is an existing file.
+
+ A sandbox that could not be reached raises: that is not an answer, and
+ a task scoring "missing" from it would record a reward of 0.
+ """
files = self._sb.files
try:
return bool(await files.exists(path)) and not bool(await files.is_dir(path))
- except Exception: # noqa: BLE001
+ except Exception as error: # noqa: BLE001
+ if _unreachable(error):
+ raise
return False
async def directory_exists(self, path: str) -> bool:
+ """Whether ``path`` is an existing directory; raises like :meth:`file_exists`."""
files = self._sb.files
try:
return bool(await files.exists(path)) and bool(await files.is_dir(path))
- except Exception: # noqa: BLE001
+ except Exception as error: # noqa: BLE001
+ if _unreachable(error):
+ raise
return False
async def read_bytes(self, path: str, offset: int = 0, length: Optional[int] = None) -> bytes:
diff --git a/libs/cua-bench/cua_bench/computers/remote.py b/libs/cua-bench/cua_bench/computers/remote.py
index 862129ed74..500bd627fa 100644
--- a/libs/cua-bench/cua_bench/computers/remote.py
+++ b/libs/cua-bench/cua_bench/computers/remote.py
@@ -1291,12 +1291,18 @@ class RemoteDesktopSession:
await self.interface.write_bytes(path, data)
async def file_exists(self, path: str) -> bool:
- """Whether ``path`` is an existing file (not a directory), as in 0.2.x."""
+ """Whether ``path`` is an existing file (not a directory), as in 0.2.x.
+
+ On a cua-sandbox session, raises if the sandbox does not answer.
+ """
await self._ensure_computer()
return await self.interface.file_exists(path)
async def directory_exists(self, path: str) -> bool:
- """Check if a directory exists in the environment."""
+ """Check if a directory exists in the environment.
+
+ On a cua-sandbox session, raises if the sandbox does not answer.
+ """
await self._ensure_computer()
return await self.interface.directory_exists(path)
diff --git a/libs/cua-bench/cua_bench/tests/test_sandbox_exists_errors.py b/libs/cua-bench/cua_bench/tests/test_sandbox_exists_errors.py
new file mode 100644
index 0000000000..35eb145aca
--- /dev/null
+++ b/libs/cua-bench/cua_bench/tests/test_sandbox_exists_errors.py
@@ -0,0 +1,88 @@
+"""file_exists / directory_exists on a sandbox session when the sandbox does not answer.
+
+A task's ``evaluate`` commonly scores "answer file missing" as 0 (the starter
+template and example_tasks/hello_file_env do). If a sandbox that stopped
+answering also reads as "missing", a frozen or crashed sandbox records a
+reward of 0 for an episode nobody scored. An unreachable sandbox has to raise.
+"""
+
+from __future__ import annotations
+
+import pytest
+from cua_bench.computers.remote import RemoteDesktopSession
+
+from .fakes import FakeSandbox
+
+
+class _FailingFiles:
+ """The files API of a sandbox whose every call fails with ``error``."""
+
+ def __init__(self, error: BaseException) -> None:
+ self.error = error
+
+ async def exists(self, path: str) -> bool:
+ raise self.error
+
+ async def is_dir(self, path: str) -> bool:
+ raise self.error
+
+
+def _session(files=None) -> RemoteDesktopSession:
+ sb = FakeSandbox()
+ if files is not None:
+ sb.files = files
+ return RemoteDesktopSession.attach(sb, os_type="linux")
+
+
+def _spacesd_not_available():
+ from cua_sandbox import SpacesdNotAvailable
+
+ return SpacesdNotAvailable("no cua-spacesd answered")
+
+
+ "error",
+ [
+ TimeoutError("operation timed out"),
+ ConnectionResetError("connection reset by peer"),
+ _spacesd_not_available(),
+ ],
+ ids=["timeout", "connection", "spacesd-not-available"],
+)
+async def test_an_unreachable_sandbox_raises_instead_of_reading_as_missing(error):
+ session = _session(_FailingFiles(error))
+ with pytest.raises(type(error)):
+ await session.file_exists("/tmp/cb_answer.txt")
+ with pytest.raises(type(error)):
+ await session.directory_exists("/tmp")
+
+
+async def test_the_native_transport_error_raises_and_not_found_reads_as_missing():
+ try:
+ from cua import CuaError
+ except Exception: # noqa: BLE001 - the native SDK library is not built here
+ pytest.skip("native cua SDK not available")
+
+ frozen = CuaError.Transport("transport error: http2 error (operation timed out)")
+ with pytest.raises(type(frozen)):
+ await _session(_FailingFiles(frozen)).file_exists("/tmp/cb_answer.txt")
+ with pytest.raises(CuaError.Timeout):
+ await _session(_FailingFiles(CuaError.Timeout("deadline exceeded"))).directory_exists("/t")
+ missing = _session(_FailingFiles(CuaError.NotFound("no such file")))
+ assert await missing.file_exists("/tmp/cb_answer.txt") is False
+
+
+async def test_present_and_missing_paths_answer_as_before():
+ session = _session()
+ await session.write_file("/tmp/cb_answer.txt", "hello")
+ assert await session.file_exists("/tmp/cb_answer.txt") is True
+ assert await session.file_exists("/tmp/missing.txt") is False
+ assert await session.file_exists("/tmp") is False
+ assert await session.directory_exists("/tmp") is True
+ assert await session.directory_exists("/tmp/missing") is False
+
+
+async def test_other_errors_still_read_as_missing():
+ session = _session(_FailingFiles(PermissionError("permission denied")))
+ assert await session.file_exists("/root/secret.txt") is False
+ assert await session.directory_exists("/root") is False