Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(cua-driver): keep browser refusal codes through protected scope attestation (#5573)
injaneityff832e5595d1
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/pointer.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/pointer.rsindex cd4fea9f64..1a046bf916 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/pointer.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/pointer.rs@@ -779,7 +779,7 @@ impl Tool for BrowserPointerTool {&self,adapter_id: &str,args: &Value,- ) -> Result<Option<Value>, String> {+ ) -> Result<Option<Value>, crate::tool::ProtectedResourceScopeError> {if adapter_id == "browser_bound_input" {browser_protected_resource_scope(&self.engine, args, "browser_pointer").await} else {diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rsindex fe23f46806..65be2ed3db 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/tools.rs@@ -295,7 +295,7 @@ pub(crate) async fn browser_protected_resource_scope(engine: &BrowserEngine,args: &Value,tool_name: &str,-) -> Result<Option<Value>, String> {+) -> Result<Option<Value>, crate::tool::ProtectedResourceScopeError> {let session = args.get("session").and_then(Value::as_str)@@ -316,8 +316,7 @@ pub(crate) async fn browser_protected_resource_scope(.unwrap_or_else(|| session.to_owned());let (validated, live_origin) = engine.attest_protected_tab(&runtime_session, target_id, tab_id)- .await- .map_err(|error| error.message)?;+ .await?;let target = validated.record;let tab = validated.tab;let requested_origin = if tool_name == "browser_navigate" {@@ -497,7 +496,7 @@ impl Tool for GetBrowserStateTool {&self,adapter_id: &str,args: &Value,- ) -> Result<Option<Value>, String> {+ ) -> Result<Option<Value>, crate::tool::ProtectedResourceScopeError> {if adapter_id == "private_observation"&& args.get("target_id").and_then(Value::as_str).is_some(){@@ -1056,7 +1055,7 @@ impl Tool for BrowserNavigateTool {&self,adapter_id: &str,args: &Value,- ) -> Result<Option<Value>, String> {+ ) -> Result<Option<Value>, crate::tool::ProtectedResourceScopeError> {if adapter_id == "browser_bound_input" {browser_protected_resource_scope(&self.engine, args, "browser_navigate").await} else {@@ -1213,7 +1212,7 @@ impl Tool for BrowserClickTool {&self,adapter_id: &str,args: &Value,- ) -> Result<Option<Value>, String> {+ ) -> Result<Option<Value>, crate::tool::ProtectedResourceScopeError> {if adapter_id == "browser_bound_input" {browser_protected_resource_scope(&self.engine, args, "browser_click").await} else {@@ -1878,7 +1877,7 @@ impl Tool for BrowserTypeTool {&self,adapter_id: &str,args: &Value,- ) -> Result<Option<Value>, String> {+ ) -> Result<Option<Value>, crate::tool::ProtectedResourceScopeError> {if adapter_id == "browser_bound_input" {browser_protected_resource_scope(&self.engine, args, "browser_type").await} else {@@ -2472,7 +2471,7 @@ impl Tool for BrowserSelectTool {&self,adapter_id: &str,args: &Value,- ) -> Result<Option<Value>, String> {+ ) -> Result<Option<Value>, crate::tool::ProtectedResourceScopeError> {if adapter_id == "browser_bound_input" {browser_protected_resource_scope(&self.engine, args, "browser_select").await} else {@@ -2671,7 +2670,7 @@ impl Tool for BrowserDialogTool {&self,adapter_id: &str,args: &Value,- ) -> Result<Option<Value>, String> {+ ) -> Result<Option<Value>, crate::tool::ProtectedResourceScopeError> {if matches!(adapter_id,"private_observation" | "browser_consequential_action"diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rsindex 9b3ab7c082..7d4b86e3cf 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/browser/v2_tests.rs@@ -2755,6 +2755,48 @@ async fn protected_browser_scope_reproves_live_origin_and_omits_sensitive_url_te);}+#[tokio::test]+async fn a_stale_binding_keeps_its_refusal_code_through_scope_attestation() {+ use crate::authorization::PermissionMode;+ use crate::session_authorization::{SessionAuthorizationRegistry, SessionModeCeiling};++ let f = fixture().await;+ let mut registry = crate::tool::ToolRegistry::new_with_protected_consent_provider(Some(+ Arc::new(FixtureProtectedProvider {+ consent_seen: AtomicBool::new(false),+ }),+ ));+ super::tools::register_browser_tools(&f.engine, &mut registry);+ let ceiling = SessionModeCeiling::for_trusted_sessions(+ [PermissionMode::Standard],+ false,+ std::time::Duration::from_secs(60),+ std::time::Duration::from_secs(30),+ )+ .unwrap();+ let context = SessionAuthorizationRegistry::with_ceiling(ceiling)+ .compatibility_context(PermissionMode::Standard, None)+ .unwrap();+ let args = json!({+ "target_id": "bt-gone",+ "tab_id": "tab-gone",+ "ref": "p1:0",+ "session": SESSION,+ });++ for tool in ["get_browser_state", "browser_click"] {+ let result = registry+ .invoke_with_context(tool, args.clone(), context.clone())+ .await;+ let refusal = &result.structured_content.unwrap()["refusal"];+ assert_eq!(refusal["code"], "browser_binding_stale", "{tool}");+ assert!(+ refusal["message"].as_str().unwrap().contains("bt-gone"),+ "{tool}"+ );+ }+}+#[tokio::test]async fn click_routes_oopif_refs_through_the_contained_child_session() {let f = fixture().await;diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/consent.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/consent.rsindex 397b15a307..782f6389a1 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/consent.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/consent.rs@@ -94,6 +94,8 @@ pub enum ConsentError {Canceled,#[error("confirmation provider failed: {0}")]Provider(String),+ #[error("{0}")]+ ProtectedScope(crate::tool::ProtectedResourceScopeError),#[error("confirmation response did not match the exact request digest")]DigestMismatch,#[error("confirmation expired before activation")]diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/perception_tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/perception_tools.rsindex d8e6b0bb14..a73e66f830 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/perception_tools.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/perception_tools.rs@@ -77,7 +77,7 @@ impl Tool for ParseVisualRegionsTool {&self,_adapter_id: &str,args: &Value,- ) -> Result<Option<Value>, String> {+ ) -> Result<Option<Value>, crate::tool::ProtectedResourceScopeError> {let binding = (self.resolve_binding)(args).map_err(|error| error.message)?;let capture_id = args.get("capture_id")diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rsindex 91c34d524f..c2b2995714 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs@@ -1105,6 +1105,42 @@ pub enum ProtectedResourceOwnership {DriverOwned,}+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct ProtectedResourceScopeError {+ pub code: Option<&'static str>,+ pub message: String,+}++impl From<String> for ProtectedResourceScopeError {+ fn from(message: String) -> Self {+ Self {+ code: None,+ message,+ }+ }+}++impl From<&str> for ProtectedResourceScopeError {+ fn from(message: &str) -> Self {+ Self::from(message.to_owned())+ }+}++impl From<crate::browser::BrowserRefusal> for ProtectedResourceScopeError {+ fn from(refusal: crate::browser::BrowserRefusal) -> Self {+ Self {+ code: Some(refusal.code.as_str()),+ message: refusal.message,+ }+ }+}++impl std::fmt::Display for ProtectedResourceScopeError {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.write_str(&self.message)+ }+}+/// A callable tool handler. Object-safe — uses `Box<dyn Tool>`.#[async_trait]pub trait Tool: Send + Sync {@@ -1139,7 +1175,7 @@ pub trait Tool: Send + Sync {&self,_adapter_id: &str,_args: &Value,- ) -> Result<Option<Value>, String> {+ ) -> Result<Option<Value>, ProtectedResourceScopeError> {Ok(None)}@@ -1150,11 +1186,11 @@ pub trait Tool: Send + Sync {adapter_id: &str,args: &Value,approved_scope: &Value,- ) -> Result<(), String> {+ ) -> Result<(), ProtectedResourceScopeError> {match self.protected_resource_scope(adapter_id, args).await? {Some(current) if current == *approved_scope => Ok(()),- Some(_) => Err("the protected resource identity changed before dispatch".to_owned()),- None => Err("the protected resource identity cannot be re-proven".to_owned()),+ Some(_) => Err("the protected resource identity changed before dispatch".into()),+ None => Err("the protected resource identity cannot be re-proven".into()),}}@@ -2700,7 +2736,7 @@ impl ToolRegistry {&& tool.protected_resource_scope("private_observation", args).await- .map_err(crate::consent::ConsentError::Provider)?+ .map_err(crate::consent::ConsentError::ProtectedScope)?.is_none(){return Err(crate::consent::ConsentError::Provider(@@ -2716,7 +2752,7 @@ impl ToolRegistry {let browser_scope = if history_resource.is_none() {tool.protected_resource_scope("private_observation", args).await- .map_err(crate::consent::ConsentError::Provider)?+ .map_err(crate::consent::ConsentError::ProtectedScope)?} else {None};@@ -3185,7 +3221,7 @@ impl ToolRegistry {let mut resource = tool.protected_resource_scope(adapter_id, args).await- .map_err(|message| protected_scope_refusal(&message))?+ .map_err(|error| protected_scope_error_refusal(&error))?.ok_or_else(|| {protected_scope_refusal("the concrete tool cannot attest an exact protected resource scope",@@ -3236,10 +3272,12 @@ impl ToolRegistry {scope.remove("driver_owned");}}- if let Err(message) = tool+ if let Err(error) = tool.validate_protected_resource_scope(adapter_id, args, &validation_scope).await{+ // Any validation error revokes the grant, whatever code it carries:+ // a CDP timeout here still means the caller must re-bind, not wait.if let Some(grant) = self.protected_resource_grants.revoke_resource(context,lifecycle_session,@@ -3249,8 +3287,8 @@ impl ToolRegistry {self.approval_broker.revoke(&grant).await;}return Err(protected_refusal(- "protected_resource_scope_stale",- &message,+ error.code.unwrap_or("protected_resource_scope_stale"),+ &error.message,));}Ok(())@@ -3901,7 +3939,7 @@ mod runtime_isolation_tests {&self,_adapter_id: &str,_args: &serde_json::Value,- ) -> Result<Option<serde_json::Value>, String> {+ ) -> Result<Option<serde_json::Value>, super::ProtectedResourceScopeError> {Ok(Some(self.scope.clone()))}@@ -3910,14 +3948,14 @@ mod runtime_isolation_tests {_adapter_id: &str,_args: &serde_json::Value,approved_scope: &serde_json::Value,- ) -> Result<(), String> {+ ) -> Result<(), super::ProtectedResourceScopeError> {if self.stale_before_dispatch {- return Err("synthetic process identity changed".to_owned());+ return Err("synthetic process identity changed".into());}if *approved_scope == self.scope {Ok(())} else {- Err("synthetic scope mismatch".to_owned())+ Err("synthetic scope mismatch".into())}}@@ -3957,6 +3995,29 @@ mod runtime_isolation_tests {}}+ struct StringScopeProbe {+ def: super::ToolDef,+ }++ #[async_trait::async_trait]+ impl super::Tool for StringScopeProbe {+ fn def(&self) -> &super::ToolDef {+ &self.def+ }++ async fn protected_resource_scope(+ &self,+ _adapter_id: &str,+ _args: &serde_json::Value,+ ) -> Result<Option<serde_json::Value>, super::ProtectedResourceScopeError> {+ Err("synthetic scope lookup failed".into())+ }++ async fn invoke(&self, _args: serde_json::Value) -> crate::protocol::ToolResult {+ crate::protocol::ToolResult::text("must not run")+ }+ }+struct AcceptingProvider {requests: AtomicUsize,}@@ -5920,6 +5981,51 @@ resources:);}+ #[tokio::test]+ async fn a_plain_string_scope_error_keeps_the_generic_refusal_codes() {+ for (name, read_only, code) in [+ ("get_browser_state", true, "authorization_host_failed"),+ ("browser_click", false, "protected_resource_scope_invalid"),+ ] {+ let mut registry = super::ToolRegistry::new_with_protected_consent_provider(Some(+ Arc::new(AcceptingProvider {+ requests: AtomicUsize::new(0),+ }),+ ));+ registry.register(Box::new(StringScopeProbe {+ def: super::ToolDef {+ name: name.to_owned(),+ description: "test string scope error".into(),+ input_schema: serde_json::json!({"type": "object"}),+ read_only,+ destructive: false,+ idempotent: read_only,+ open_world: true,+ },+ }));+ let result = registry+ .invoke_with_context(+ name,+ serde_json::json!({+ "target_id": "target-1",+ "tab_id": "tab-1",+ "session": "browser"+ }),+ standard_context(),+ )+ .await;+ assert_eq!(+ result+ .structured_content+ .as_ref()+ .and_then(|value| value.pointer("/refusal/code"))+ .and_then(serde_json::Value::as_str),+ Some(code),+ "{name}"+ );+ }+ }+#[tokio::test]async fn browser_observation_without_a_live_origin_attestation_fails_closed() {let hits = Arc::new(AtomicUsize::new(0));@@ -6606,6 +6712,9 @@ fn protected_consent_refusal(error: crate::consent::ConsentError) -> ToolResultcrate::consent::ConsentError::DigestMismatch => "authorization_scope_mismatch",crate::consent::ConsentError::Expired => "authorization_expired",crate::consent::ConsentError::Provider(_) => "authorization_host_failed",+ crate::consent::ConsentError::ProtectedScope(error) => {+ error.code.unwrap_or("authorization_host_failed")+ }crate::consent::ConsentError::BoundedResource(_) => "bounded_resource_outside_manifest",};let message = error.to_string();@@ -6622,6 +6731,13 @@ fn protected_scope_refusal(message: &str) -> ToolResult {protected_refusal("protected_resource_scope_invalid", message)}+fn protected_scope_error_refusal(error: &ProtectedResourceScopeError) -> ToolResult {+ protected_refusal(+ error.code.unwrap_or("protected_resource_scope_invalid"),+ &error.message,+ )+}+/// Recovery text for a call on an ended lifecycle episode. A named session is/// restarted by name from its own transport; an unnamed one has no label to/// pass, so naming one would only start an unrelated session.diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/window_target.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/window_target.rsindex 2da68ec6f3..fec7c8ce00 100644--- a/libs/cua-driver/rust/crates/cua-driver-core/src/window_target.rs+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/window_target.rs@@ -163,7 +163,7 @@ impl Tool for PidOnlyWindowTargetGuard {&self,adapter_id: &str,args: &Value,- ) -> Result<Option<Value>, String> {+ ) -> Result<Option<Value>, crate::tool::ProtectedResourceScopeError> {self.inner.protected_resource_scope(adapter_id, args).await}@@ -172,7 +172,7 @@ impl Tool for PidOnlyWindowTargetGuard {adapter_id: &str,args: &Value,approved_scope: &Value,- ) -> Result<(), String> {+ ) -> Result<(), crate::tool::ProtectedResourceScopeError> {self.inner.validate_protected_resource_scope(adapter_id, args, approved_scope).awaitdiff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/kill_app.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/kill_app.rsindex d06ed5d16b..72b6da2810 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/kill_app.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/kill_app.rs@@ -33,7 +33,7 @@ impl Tool for KillAppTool {&self,adapter_id: &str,args: &Value,- ) -> Result<Option<Value>, String> {+ ) -> Result<Option<Value>, cua_driver_core::tool::ProtectedResourceScopeError> {if adapter_id != "process_control" {return Ok(None);}@@ -61,7 +61,7 @@ impl Tool for KillAppTool {{Ok(Some(scope)) => scope["fingerprint"].clone(),Ok(None) => Value::Null,- Err(message) => return kill_app_stale_process_refusal(message),+ Err(error) => return kill_app_stale_process_refusal(error.message),};if current != *expected {return kill_app_stale_process_refusal(diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/kill_app.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/kill_app.rsindex c7c75e9072..2b06cf254e 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/kill_app.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/kill_app.rs@@ -55,7 +55,7 @@ impl Tool for KillAppTool {&self,adapter_id: &str,args: &Value,- ) -> Result<Option<Value>, String> {+ ) -> Result<Option<Value>, cua_driver_core::tool::ProtectedResourceScopeError> {if adapter_id != "process_control" {return Ok(None);}@@ -82,8 +82,8 @@ impl Tool for KillAppTool {{Ok(Some(scope)) => scope["fingerprint"].clone(),Ok(None) => Value::Null,- Err(message) => {- return stale_process_refusal(message);+ Err(error) => {+ return stale_process_refusal(error.message);}};if current != *expected {diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex 946914ee56..c5d198335a 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -476,7 +476,7 @@ impl Tool for ExactPidWindowTargetGuard {&self,adapter_id: &str,args: &Value,- ) -> Result<Option<Value>, String> {+ ) -> Result<Option<Value>, cua_driver_core::tool::ProtectedResourceScopeError> {self.inner.protected_resource_scope(adapter_id, args).await}@@ -485,7 +485,7 @@ impl Tool for ExactPidWindowTargetGuard {adapter_id: &str,args: &Value,approved_scope: &Value,- ) -> Result<(), String> {+ ) -> Result<(), cua_driver_core::tool::ProtectedResourceScopeError> {self.inner.validate_protected_resource_scope(adapter_id, args, approved_scope).awaitdiff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/kill_app.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/kill_app.rsindex 14a29b61f6..d23250de52 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/kill_app.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/kill_app.rs@@ -50,7 +50,7 @@ impl Tool for KillAppTool {&self,adapter_id: &str,args: &Value,- ) -> Result<Option<Value>, String> {+ ) -> Result<Option<Value>, cua_driver_core::tool::ProtectedResourceScopeError> {if adapter_id != "process_control" {return Ok(None);}@@ -78,7 +78,7 @@ impl Tool for KillAppTool {{Ok(Some(scope)) => scope["fingerprint"].clone(),Ok(None) => Value::Null,- Err(message) => return kill_app_stale_process_refusal(message),+ Err(error) => return kill_app_stale_process_refusal(error.message),};if current != *expected {return kill_app_stale_process_refusal(