Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): resolve the perception extension per parse on a live runtime (#4197)

Francesco Bonaccie8777e6d6a99
diff --git a/docs/content/docs/how-to-guides/driver/parse-visual-regions.mdx b/docs/content/docs/how-to-guides/driver/parse-visual-regions.mdx
index 1be4d04ab7..8c4d927390 100644
--- a/docs/content/docs/how-to-guides/driver/parse-visual-regions.mdx
+++ b/docs/content/docs/how-to-guides/driver/parse-visual-regions.mdx
@@ -79,6 +79,10 @@ Installation is never triggered by capture, parsing, Driver startup, or an
update check. A parse on a default installation returns `not_installed`; it does
not download a worker or model.
+You do not need to restart Cua Driver after installing. An already running
+daemon or MCP session reverifies the installed extension on its next
+`parse_visual_regions` call.
+
## Capture a fresh target
Use one persistent MCP connection or one typed SDK runtime for capture, parse,
@@ -209,6 +213,10 @@ cua-driver extension remove cua-perception
Removal does not alter the default Driver or external `cua-som` installations.
+Update and removal also take effect on a running Driver without a restart. A
+parse that is already running finishes with the version it started with; the
+next parse uses the updated version or returns `not_installed`.
+
## Use an external chooser
To put `jev-use` or another decision provider above this loop, follow [Use
diff --git a/libs/cua-driver/docs/perception-extension.md b/libs/cua-driver/docs/perception-extension.md
index e014deb1b8..c81c5eb176 100644
--- a/libs/cua-driver/docs/perception-extension.md
+++ b/libs/cua-driver/docs/perception-extension.md
@@ -126,6 +126,12 @@ Inspect a replacement before running `extension update`. Remove only the
extension-owned installation with `extension remove cua-perception`. Neither
operation changes the default Driver or an external `cua-som` installation.
+Install, update, and remove take effect on a running Driver without a restart.
+A running daemon or `cua-driver mcp` session checks the installed extension
+before each `parse_visual_regions` call and fully reverifies it whenever the
+installation changed. A parse that is already running finishes with the version
+it started with.
+
## Capture-bound parse and action
Keep capture, parse, action, and reobservation on one persistent MCP connection
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/perception_client.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/perception_client.rs
index a035e95e27..ef326c23cf 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/perception_client.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/perception_client.rs
@@ -188,6 +188,22 @@ struct PerceptionState {
cancellation_notify: Notify,
}
+impl PerceptionState {
+ fn shutdown_now(&self) {
+ self.cancellation_epoch.fetch_add(1, Ordering::AcqRel);
+ self.cancellation_notify.notify_waiters();
+ self.drop_idle_worker();
+ }
+
+ /// Drop the warm worker only when no request currently owns it. An
+ /// in-flight request keeps its worker and finishes normally.
+ fn drop_idle_worker(&self) {
+ if let Ok(mut worker) = self.worker.try_lock() {
+ worker.take();
+ }
+ }
+}
+
impl Drop for PerceptionState {
fn drop(&mut self) {
self.worker.get_mut().take();
@@ -259,13 +275,14 @@ impl PerceptionClient {
self.config.is_some()
}
+ /// The verified worker configuration, when an extension is installed.
+ pub fn worker_config(&self) -> Option<&PerceptionWorkerConfig> {
+ self.config.as_deref()
+ }
+
/// Cancel any in-flight request and synchronously drop an idle worker.
pub fn shutdown_now(&self) {
- self.state.cancellation_epoch.fetch_add(1, Ordering::AcqRel);
- self.state.cancellation_notify.notify_waiters();
- if let Ok(mut worker) = self.state.worker.try_lock() {
- worker.take();
- }
+ self.state.shutdown_now();
}
/// Launch one worker process and issue exactly one parse. Failures are
@@ -420,6 +437,169 @@ impl PerceptionClient {
}
}
+/// Cheap change detection plus full verification for an installed worker.
+///
+/// A live Driver consults the fingerprint before every parse and calls
+/// [`resolve`](Self::resolve) only when it changes, so install, update, and
+/// removal take effect without a restart while full verification still runs
+/// for every newly observed installation state.
+pub trait PerceptionClientResolver: Send + Sync + 'static {
+ /// Return a token that changes whenever install, update, or removal could
+ /// change the resolved client. This must be cheap: it runs on every parse
+ /// and must not hash payloads.
+ fn fingerprint(&self) -> String;
+
+ /// Fully verify the installed extension and build its client. A missing or
+ /// unusable installation yields [`PerceptionClient::unavailable`].
+ fn resolve(&self) -> PerceptionClient;
+}
+
+/// The perception client a registered `parse_visual_regions` tool uses.
+///
+/// A fixed handle always returns the same client. A live handle re-resolves
+/// its client whenever the resolver's fingerprint changes. Replacing a client
+/// never cancels a request already running on it: that request holds its own
+/// reference and finishes against the version it started with, after which
+/// the replaced worker is released.
+#[derive(Clone)]
+pub struct PerceptionClientHandle {
+ source: Arc<PerceptionClientSource>,
+}
+
+enum PerceptionClientSource {
+ Fixed(PerceptionClient),
+ Live(LivePerceptionClient),
+}
+
+struct LivePerceptionClient {
+ resolver: Arc<dyn PerceptionClientResolver>,
+ /// Serializes full verification so concurrent parses verify one change once.
+ resolving: std::sync::Mutex<()>,
+ state: std::sync::Mutex<LivePerceptionState>,
+}
+
+#[derive(Default)]
+struct LivePerceptionState {
+ current: Option<(String, PerceptionClient)>,
+ /// Replaced clients that may still serve an in-flight request, retained
+ /// weakly so a session end can still cancel that request.
+ retired: Vec<std::sync::Weak<PerceptionState>>,
+}
+
+impl PerceptionClientHandle {
+ pub fn fixed(client: PerceptionClient) -> Self {
+ Self {
+ source: Arc::new(PerceptionClientSource::Fixed(client)),
+ }
+ }
+
+ pub fn live(resolver: Arc<dyn PerceptionClientResolver>) -> Self {
+ Self {
+ source: Arc::new(PerceptionClientSource::Live(LivePerceptionClient {
+ resolver,
+ resolving: std::sync::Mutex::new(()),
+ state: std::sync::Mutex::new(LivePerceptionState::default()),
+ })),
+ }
+ }
+
+ /// Return the client for the currently installed extension. A live handle
+ /// may perform blocking verification, so async callers use
+ /// [`current`](Self::current).
+ pub fn current_blocking(&self) -> PerceptionClient {
+ match self.source.as_ref() {
+ PerceptionClientSource::Fixed(client) => client.clone(),
+ PerceptionClientSource::Live(live) => live.current(),
+ }
+ }
+
+ /// Return the client for the currently installed extension, running any
+ /// required verification off the async executor.
+ pub async fn current(&self) -> PerceptionClient {
+ match self.source.as_ref() {
+ PerceptionClientSource::Fixed(client) => client.clone(),
+ PerceptionClientSource::Live(_) => {
+ let handle = self.clone();
+ tokio::task::spawn_blocking(move || handle.current_blocking())
+ .await
+ .unwrap_or_else(|error| {
+ tracing::warn!("perception client resolution task failed: {error}");
+ PerceptionClient::unavailable()
+ })
+ }
+ }
+ }
+
+ /// Cancel in-flight requests and drop idle workers for the current client
+ /// and for any replaced client still serving a request.
+ pub fn shutdown_now(&self) {
+ match self.source.as_ref() {
+ PerceptionClientSource::Fixed(client) => client.shutdown_now(),
+ PerceptionClientSource::Live(live) => {
+ let mut state = live.lock_state();
+ if let Some((_, client)) = &state.current {
+ client.shutdown_now();
+ }
+ state.retired.retain(|retired| match retired.upgrade() {
+ Some(retired) => {
+ retired.shutdown_now();
+ true
+ }
+ None => false,
+ });
+ }
+ }
+ }
+}
+
+impl From<PerceptionClient> for PerceptionClientHandle {
+ fn from(client: PerceptionClient) -> Self {
+ Self::fixed(client)
+ }
+}
+
+impl LivePerceptionClient {
+ fn lock_state(&self) -> std::sync::MutexGuard<'_, LivePerceptionState> {
+ self.state
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ }
+
+ fn cached(&self, fingerprint: &str) -> Option<PerceptionClient> {
+ self.lock_state()
+ .current
+ .as_ref()
+ .filter(|(cached, _)| cached == fingerprint)
+ .map(|(_, client)| client.clone())
+ }
+
+ fn current(&self) -> PerceptionClient {
+ let fingerprint = self.resolver.fingerprint();
+ if let Some(client) = self.cached(&fingerprint) {
+ return client;
+ }
+ let _resolving = self
+ .resolving
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ // Another caller may have verified this state while we waited.
+ let fingerprint = self.resolver.fingerprint();
+ if let Some(client) = self.cached(&fingerprint) {
+ return client;
+ }
+ // The fingerprint is sampled before verification, so a change racing
+ // with resolution is observed again by the next request.
+ let client = self.resolver.resolve();
+ let mut state = self.lock_state();
+ if let Some((_, replaced)) = state.current.replace((fingerprint, client.clone())) {
+ replaced.state.drop_idle_worker();
+ state.retired.retain(|retired| retired.strong_count() > 0);
+ state.retired.push(Arc::downgrade(&replaced.state));
+ }
+ client
+ }
+}
+
struct WarmWorker {
contained: ContainedChild,
/// Taken to close the request pipe when the worker should observe end of
@@ -1098,6 +1278,8 @@ request = read_frame()
if counter:
with open(counter, 'a', encoding='utf-8') as handle:
handle.write('parse\n')
+if mode == 'slow':
+ time.sleep(1.0)
if mode == 'hang':
time.sleep(60)
elif mode == 'crash':
@@ -1852,6 +2034,73 @@ write_frame({'protocol':'cua-perception/1','request_id':request['request_id'],'s
assert_eq!(std::fs::read_to_string(counter).unwrap(), "parse\n");
}
+ struct FixtureResolver(std::sync::Mutex<(String, PerceptionClient)>);
+
+ impl PerceptionClientResolver for FixtureResolver {
+ fn fingerprint(&self) -> String {
+ self.0.lock().unwrap().0.clone()
+ }
+
+ fn resolve(&self) -> PerceptionClient {
+ self.0.lock().unwrap().1.clone()
+ }
+ }
+
+ #[tokio::test]
+ async fn live_handle_replacement_does_not_break_an_in_flight_parse() {
+ let counter_directory = tempfile::tempdir().unwrap();
+ let counter = counter_directory.path().join("count");
+ let (directory, worker) = fixture_worker("slow", Some(&counter));
+ let installed = client(&directory, worker, Duration::from_secs(10), 1024 * 1024);
+ let resolver = Arc::new(FixtureResolver(std::sync::Mutex::new((
+ "v1".to_owned(),
+ installed,
+ ))));
+ let handle = PerceptionClientHandle::live(resolver.clone());
+
+ let in_flight = {
+ let handle = handle.clone();
+ tokio::spawn(async move {
+ handle
+ .current()
+ .await
+ .parse(
+ "capture-test",
+ 2,
+ 2,
+ &[1, 2, 3, 4],
+ &PerceptionCancellation::default(),
+ )
+ .await
+ })
+ };
+ for _ in 0..500 {
+ if std::fs::read_to_string(&counter).is_ok_and(|calls| calls.contains("parse")) {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ assert!(counter.exists(), "the first parse must reach the worker");
+
+ *resolver.0.lock().unwrap() = ("removed".to_owned(), PerceptionClient::unavailable());
+ let replaced = handle
+ .current()
+ .await
+ .parse(
+ "capture-test",
+ 2,
+ 2,
+ &[1, 2, 3, 4],
+ &PerceptionCancellation::default(),
+ )
+ .await
+ .unwrap_err();
+ assert_eq!(replaced.code, VisualParseErrorCode::NotInstalled);
+
+ let result = in_flight.await.unwrap().unwrap();
+ assert_eq!(result["runtime"], "fixture");
+ }
+
#[tokio::test]
async fn malformed_response_is_not_replayed() {
let counter_directory = tempfile::tempdir().unwrap();
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/perception_tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/perception_tools.rs
index a3f066275e..a0407729a3 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/perception_tools.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/perception_tools.rs
@@ -16,7 +16,7 @@ use cua_driver_contract::{
use crate::capture_registry::{CaptureLookupError, PerceptionCapture};
use crate::capture_runtime::{CaptureBinding, CaptureService, CaptureTarget};
-use crate::perception_client::{error, PerceptionCancellation, PerceptionClient};
+use crate::perception_client::{error, PerceptionCancellation, PerceptionClientHandle};
use crate::protocol::ToolResult;
use crate::tool::{Tool, ToolDef, ToolRegistry};
use crate::tool_args::parse_typed_projection;
@@ -26,9 +26,10 @@ pub type CaptureBindingResolver =
pub fn register_perception_tool(
registry: &mut ToolRegistry,
- client: PerceptionClient,
+ client: impl Into<PerceptionClientHandle>,
resolve_binding: CaptureBindingResolver,
) {
+ let client = client.into();
let shutdown_client = client.clone();
registry.retain_session_end_hook(crate::session::register_scoped_session_end_hook(
move |_| shutdown_client.shutdown_now(),
@@ -43,14 +44,16 @@ pub fn register_perception_tool(
struct ParseVisualRegionsTool {
def: ToolDef,
captures: Arc<CaptureService>,
- client: PerceptionClient,
+ /// Resolved per request so extension install, update, and removal take
+ /// effect on a running Driver.
+ client: PerceptionClientHandle,
resolve_binding: CaptureBindingResolver,
}
impl ParseVisualRegionsTool {
fn new(
captures: Arc<CaptureService>,
- client: PerceptionClient,
+ client: impl Into<PerceptionClientHandle>,
resolve_binding: CaptureBindingResolver,
) -> Self {
let contract = cua_driver_contract::tool_contract(ParseVisualRegionsInput::TOOL_NAME)
@@ -58,7 +61,7 @@ impl ParseVisualRegionsTool {
Self {
def: ToolDef::from_contract(&contract),
captures,
- client,
+ client: client.into(),
resolve_binding,
}
}
@@ -132,10 +135,10 @@ impl Tool for ParseVisualRegionsTool {
Ok(capture) => capture,
Err(cause) => return tool_error(map_capture_error(cause)),
};
+ let client = self.client.current().await;
let started = Instant::now();
let dimensions = capture.encoded_dimensions();
- let result = self
- .client
+ let result = client
.parse(
&input.capture_id,
dimensions.width(),
@@ -559,6 +562,7 @@ mod tests {
ScreenshotToActionTransform,
};
use crate::image_utils::encode_rgba_to_png;
+ use crate::perception_client::{PerceptionClient, PerceptionClientResolver};
use sha2::Digest as _;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
@@ -752,6 +756,76 @@ mod tests {
);
}
+ /// A resolver whose installation state the test flips between absent and
+ /// an installed worker, counting how often full verification runs.
+ struct SwitchableResolver {
+ installed: std::sync::Mutex<Option<String>>,
+ resolutions: AtomicU64,
+ }
+
+ impl PerceptionClientResolver for SwitchableResolver {
+ fn fingerprint(&self) -> String {
+ format!("{:?}", self.installed.lock().unwrap())
+ }
+
+ fn resolve(&self) -> PerceptionClient {
+ self.resolutions.fetch_add(1, Ordering::SeqCst);
+ match self.installed.lock().unwrap().as_deref() {
+ Some(executable) => PerceptionClient::new(
+ crate::perception_client::PerceptionWorkerConfig::new(executable),
+ )
+ .unwrap(),
+ None => PerceptionClient::unavailable(),
+ }
+ }
+ }
+
+ #[tokio::test]
+ async fn live_client_observes_install_and_removal_without_reregistration() {
+ let service = Arc::new(CaptureService::default());
+ let (capture_id, binding) = capture(&service);
+ let resolver = Arc::new(SwitchableResolver {
+ installed: std::sync::Mutex::new(None),
+ resolutions: AtomicU64::new(0),
+ });
+ let tool = ParseVisualRegionsTool::new(
+ service,
+ PerceptionClientHandle::live(resolver.clone()),
+ Arc::new(move |_| Ok(binding.clone())),
+ );
+ let parse = || tool.invoke(json!({"capture_id": capture_id.clone()}));
+
+ let absent = parse().await;
+ assert_eq!(absent.structured_content.unwrap()["code"], "not_installed");
+ let absent_again = parse().await;
+ assert_eq!(
+ absent_again.structured_content.unwrap()["code"],
+ "not_installed"
+ );
+ assert_eq!(
+ resolver.resolutions.load(Ordering::SeqCst),
+ 1,
+ "an unchanged installation must not be re-verified"
+ );
+
+ let missing_worker = std::env::temp_dir()
+ .join("cua-perception-live-reload-test")
+ .join("definitely-not-a-worker");
+ *resolver.installed.lock().unwrap() = Some(missing_worker.display().to_string());
+ let installed = parse().await;
+ let code = installed.structured_content.unwrap()["code"].clone();
+ assert_ne!(
+ code, "not_installed",
+ "the same registered tool must reach the newly installed worker"
+ );
+ assert_eq!(resolver.resolutions.load(Ordering::SeqCst), 2);
+
+ *resolver.installed.lock().unwrap() = None;
+ let removed = parse().await;
+ assert_eq!(removed.structured_content.unwrap()["code"], "not_installed");
+ assert_eq!(resolver.resolutions.load(Ordering::SeqCst), 3);
+ }
+
#[tokio::test]
async fn expired_capture_fails_at_the_tool_boundary_before_worker_launch() {
let clock = Arc::new(ManualClock::default());
diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
index 5aedb222cd..054d0ba938 100644
--- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs
@@ -875,7 +875,10 @@ impl ToolRegistry {
self.register(Box::new(EndSessionTool));
}
- pub fn register_perception_tool(&mut self, client: crate::perception_client::PerceptionClient) {
+ pub fn register_perception_tool(
+ &mut self,
+ client: impl Into<crate::perception_client::PerceptionClientHandle>,
+ ) {
let captures = self.capture_service();
let resolve_binding = Arc::new(move |args: &Value| {
captures.binding_from_args(args).map_err(|error| {
@@ -892,7 +895,7 @@ impl ToolRegistry {
pub fn register_perception_tool_with_binding_resolver(
&mut self,
- client: crate::perception_client::PerceptionClient,
+ client: impl Into<crate::perception_client::PerceptionClientHandle>,
resolve_binding: crate::perception_tools::CaptureBindingResolver,
) {
crate::perception_tools::register_perception_tool(self, client, resolve_binding);
diff --git a/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs
index 746df28e48..1dec7d14cf 100644
--- a/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs
+++ b/libs/cua-driver/rust/crates/cua-driver-sdk/src/lib.rs
@@ -53,22 +53,28 @@ use runtime::RuntimeOptions;
use service_session::ServiceSessionClient;
use worker::{ActionCompletion, PrivateWorkerClient};
-type PerceptionClientProvider = fn() -> cua_driver_core::perception_client::PerceptionClient;
-static PERCEPTION_CLIENT_PROVIDER: OnceLock<PerceptionClientProvider> = OnceLock::new();
+static PERCEPTION_CLIENT_RESOLVER: OnceLock<
+ Arc<dyn cua_driver_core::perception_client::PerceptionClientResolver>,
+> = OnceLock::new();
/// Configure the binary-owned resolver for optional installed perception.
-/// Language SDKs without a binary host retain the typed unavailable tool.
+/// Every runtime consults it per `parse_visual_regions` request, so extension
+/// install, update, and removal take effect on a running Driver. Language SDKs
+/// without a binary host retain the typed unavailable tool.
#[doc(hidden)]
-pub fn configure_perception_client_provider(provider: PerceptionClientProvider) {
- let _ = PERCEPTION_CLIENT_PROVIDER.set(provider);
+pub fn configure_perception_client_resolver(
+ resolver: Arc<dyn cua_driver_core::perception_client::PerceptionClientResolver>,
+) {
+ let _ = PERCEPTION_CLIENT_RESOLVER.set(resolver);
}
-pub(crate) fn configured_perception_client() -> cua_driver_core::perception_client::PerceptionClient
-{
- PERCEPTION_CLIENT_PROVIDER
- .get()
- .map(|provider| provider())
- .unwrap_or_else(cua_driver_core::perception_client::PerceptionClient::unavailable)
+pub(crate) fn configured_perception_client(
+) -> cua_driver_core::perception_client::PerceptionClientHandle {
+ use cua_driver_core::perception_client::{PerceptionClient, PerceptionClientHandle};
+ PERCEPTION_CLIENT_RESOLVER.get().map_or_else(
+ || PerceptionClientHandle::fixed(PerceptionClient::unavailable()),
+ |resolver| PerceptionClientHandle::live(resolver.clone()),
+ )
}
fn host_sessions_json_for_prefix(runtime_prefix: &str) -> Value {
diff --git a/libs/cua-driver/rust/crates/cua-driver/src/extension_manager.rs b/libs/cua-driver/rust/crates/cua-driver/src/extension_manager.rs
index 003fa04568..bc5a895437 100644
--- a/libs/cua-driver/rust/crates/cua-driver/src/extension_manager.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/src/extension_manager.rs
@@ -30,7 +30,9 @@ use cua_driver_core::perception_client::containment::{
use cua_driver_core::perception_client::containment::{
run_contained_hook, ContainmentLimits, HookOutcome,
};
-use cua_driver_core::perception_client::{PerceptionClient, PerceptionWorkerConfig};
+use cua_driver_core::perception_client::{
+ PerceptionClient, PerceptionClientResolver, PerceptionWorkerConfig,
+};
use cua_driver_core::protocol::ToolResult;
use cua_driver_core::tool::{Tool, ToolDef, ToolRegistry};
@@ -1021,8 +1023,8 @@ fn extension_root() -> Result<PathBuf> {
/// Resolve the optional perception worker through the same active-pointer and
/// installed-payload verification used by extension lifecycle commands.
-pub(crate) fn perception_client() -> PerceptionClient {
- match perception_worker_config() {
+fn perception_client_in(store: &ExtensionStore) -> PerceptionClient {
+ match perception_worker_config_in(store) {
Ok(Some(config)) => PerceptionClient::new(config).unwrap_or_else(|error| {
tracing::warn!("installed cua-perception configuration is unusable: {error:?}");
PerceptionClient::unavailable()
@@ -1035,6 +1037,106 @@ pub(crate) fn perception_client() -> PerceptionClient {
}
}
+/// Resolver a running Driver consults before every `parse_visual_regions`
+/// request. It fingerprints the installation cheaply and re-runs the full
+/// startup verification only when install, update, or removal changed it.
+pub(crate) fn perception_client_resolver() -> std::sync::Arc<dyn PerceptionClientResolver> {
+ std::sync::Arc::new(InstalledPerceptionResolver { root: None })
+}
+
+struct InstalledPerceptionResolver {
+ /// Fixed extension root for tests; production follows the environment.
+ root: Option<PathBuf>,
+}
+
+impl InstalledPerceptionResolver {
+ fn store(&self) -> Result<ExtensionStore> {
+ match &self.root {
+ Some(root) => Ok(ExtensionStore::new(root.clone())),
+ None => Ok(ExtensionStore::new(extension_root()?)),
+ }
+ }
+}
+
+impl PerceptionClientResolver for InstalledPerceptionResolver {
+ fn fingerprint(&self) -> String {
+ match self.store() {
+ Ok(store) => perception_install_fingerprint(&store),
+ Err(error) => format!("unresolved-root:{error:#}"),
+ }
+ }
+
+ fn resolve(&self) -> PerceptionClient {
+ match self.store() {
+ Ok(store) => perception_client_in(&store),
+ Err(error) => {
+ tracing::warn!("installed cua-perception extension is unavailable: {error:#}");
+ PerceptionClient::unavailable()
+ }
+ }
+ }
+}
+
+/// Cheap change detector for the installed perception extension.
+///
+/// Every lifecycle operation that changes what the worker resolves to renames,
+/// creates, or removes the activation pointer, the extension directory, the
+/// installed version's manifest and ownership record, or the publisher trust
+/// record. Their metadata and the pointer's contents change with it, so this
+/// never hashes payloads; a changed fingerprint triggers full verification.
+fn perception_install_fingerprint(store: &ExtensionStore) -> String {
+ fn metadata_token(path: &Path) -> String {
+ match fs::symlink_metadata(path) {
+ Ok(metadata) => {
+ #[cfg(unix)]
+ let identity = {
+ use std::os::unix::fs::MetadataExt as _;
+ format!("{}:{}", metadata.dev(), metadata.ino())
+ };
+ #[cfg(not(unix))]
+ let identity = String::new();
+ format!(
+ "{:?}:{}:{:?}:{:?}:{identity}",
+ metadata.file_type(),
+ metadata.len(),
+ metadata.modified().ok(),
+ metadata.created().ok(),
+ )
+ }
+ Err(error) => format!("absent:{:?}", error.kind()),
+ }
+ }
+
+ let extension = store.extension_dir(PERCEPTION_ID);
+ let active = extension.join(ACTIVE_NAME);
+ let mut parts = vec![
+ metadata_token(&store.root),
+ metadata_token(&store.root.join(TRUST_NAME)),
+ metadata_token(&extension),
+ metadata_token(&active),
+ metadata_token(&extension.join(ACTIVE_BACKUP_NAME)),
+ metadata_token(&extension.join(ACTIVE_NEW_NAME)),
+ ];
+ let pointer = fs::symlink_metadata(&active)
+ .ok()
+ .filter(|metadata| metadata.is_file() && metadata.len() <= MAX_IN_MEMORY_BYTES)
+ .and_then(|_| fs::read(&active).ok());
+ if let Some(bytes) = pointer {
+ parts.push(hex_sha256(&bytes));
+ if let Some(version) = serde_json::from_slice::<ActivePointer>(&bytes)
+ .ok()
+ .filter(|pointer| validate_version_segment(&pointer.version).is_ok())
+ .map(|pointer| pointer.version)
+ {
+ let version = store.versions_dir(PERCEPTION_ID).join(version);
+ parts.push(metadata_token(&version));
+ parts.push(metadata_token(&version.join(MANIFEST_NAME)));
+ parts.push(metadata_token(&version.join(INSTALL_RECORD_NAME)));
+ }
+ }
+ parts.join("|")
+}
+
pub(crate) fn perception_client_for_cli() -> Result<PerceptionClient> {
match perception_worker_config()? {
Some(config) => PerceptionClient::new(config).map_err(|error| {
@@ -5251,6 +5353,88 @@ mod tests {
path
}
+ #[tokio::test]
+ async fn live_perception_client_follows_install_update_and_remove() {
+ use cua_driver_contract::VisualParseErrorCode;
+ use cua_driver_core::perception_client::{PerceptionCancellation, PerceptionClientHandle};
+
+ async fn parse_code(handle: &PerceptionClientHandle) -> VisualParseErrorCode {
+ handle
+ .current()
+ .await
+ .parse(
+ "capture-test",
+ 1,
+ 1,
+ &[1, 2, 3, 4],
+ &PerceptionCancellation::default(),
+ )
+ .await
+ .expect_err("the fixture payload is not a runnable worker")
+ .code
+ }
+ fn active_version(client: &PerceptionClient) -> Option<String> {
+ client
+ .worker_config()
+ .and_then(|config| config.args.last().cloned())
+ }
+
+ let temp = TempDir::new().unwrap();
+ let root = temp.path().join("extensions");
+ let store = ExtensionStore::new(root.clone());
+ let entry = registry_entry(PERCEPTION_ID).unwrap();
+ let runtime_name = if cfg!(target_os = "windows") {
+ "onnxruntime.dll"
+ } else if cfg!(target_os = "macos") {
+ "libonnxruntime.dylib"
+ } else {
+ "libonnxruntime.so"
+ };
+ // One handle stands in for a runtime built before the extension existed.
+ let resolver = std::sync::Arc::new(InstalledPerceptionResolver {
+ root: Some(root.clone()),
+ });
+ let handle = PerceptionClientHandle::live(resolver.clone());
+
+ assert!(!handle.current().await.is_available());
+ assert_eq!(
+ parse_code(&handle).await,
+ VisualParseErrorCode::NotInstalled
+ );
+ let absent = resolver.fingerprint();
+ assert_eq!(resolver.fingerprint(), absent, "fingerprint must be stable");
+
+ let first = perception_fixture_archive(temp.path(), "1.0.0", runtime_name);
+ store.install_archive(entry, &first).unwrap();
+ let installed = handle.current().await;
+ assert_eq!(active_version(&installed).as_deref(), Some("1.0.0"));
+ assert_ne!(
+ parse_code(&handle).await,
+ VisualParseErrorCode::NotInstalled,
+ "a running runtime must reach the newly installed worker"
+ );
+ let installed_fingerprint = resolver.fingerprint();
+ assert_ne!(installed_fingerprint, absent);
+ assert_eq!(resolver.fingerprint(), installed_fingerprint);
+
+ let second_directory = temp.path().join("update");
+ fs::create_dir_all(&second_directory).unwrap();
+ let second = perception_fixture_archive(&second_directory, "1.1.0", runtime_name);
+ store.install_archive(entry, &second).unwrap();
+ assert_ne!(resolver.fingerprint(), installed_fingerprint);
+ let updated = handle.current().await;
+ assert_eq!(active_version(&updated).as_deref(), Some("1.1.0"));
+ // The pre-update client stays usable for a request that already holds it.
+ assert_eq!(active_version(&installed).as_deref(), Some("1.0.0"));
+
+ store.remove(entry).unwrap();
+ assert!(!handle.current().await.is_available());
+ assert_eq!(
+ parse_code(&handle).await,
+ VisualParseErrorCode::NotInstalled
+ );
+ }
+
#[test]
fn absent_perception_extension_has_no_worker_config() {
let temp = TempDir::new().unwrap();
diff --git a/libs/cua-driver/rust/crates/cua-driver/src/main.rs b/libs/cua-driver/rust/crates/cua-driver/src/main.rs
index 2531186727..2dca194133 100644
--- a/libs/cua-driver/rust/crates/cua-driver/src/main.rs
+++ b/libs/cua-driver/rust/crates/cua-driver/src/main.rs
@@ -465,7 +465,9 @@ mod mcp_runtime_selection_tests {
#[cfg(target_os = "macos")]
fn main() {
- cua_driver_sdk::configure_perception_client_provider(extension_manager::perception_client);
+ cua_driver_sdk::configure_perception_client_resolver(
+ extension_manager::perception_client_resolver(),
+ );
if let Some(code) = platform_macos::permissions::gate::run_permission_probe_if_requested() {
std::process::exit(code);
}
@@ -897,7 +899,9 @@ fn main() {
#[cfg(not(target_os = "macos"))]
fn main() -> anyhow::Result<()> {
- cua_driver_sdk::configure_perception_client_provider(extension_manager::perception_client);
+ cua_driver_sdk::configure_perception_client_resolver(
+ extension_manager::perception_client_resolver(),
+ );
if let Some(code) = history_runtime::run_offline_purge_if_requested() {
std::process::exit(code);
}