Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(cua-driver): capture X11 DirectColor window colors (#3758)

Peter Steinberger0dd44d62226f
diff --git a/.github/scripts/tests/test_ci_rust_linux_capture.py b/.github/scripts/tests/test_ci_rust_linux_capture.py
index f794eb0fe7..8786717a1c 100644
--- a/.github/scripts/tests/test_ci_rust_linux_capture.py
+++ b/.github/scripts/tests/test_ci_rust_linux_capture.py
@@ -56,7 +56,7 @@ class CaptureCiTests(unittest.TestCase):
def test_readiness_probes_do_not_reset_disposable_servers(self):
servers = [shlex.split(line) for line in capture_script().splitlines()
if line.startswith("Xvfb ")]
- self.assertEqual(len(servers), 2)
+ self.assertEqual(len(servers), 3) # primary, secondary, DirectColor (:100)
self.assertTrue(all("-noreset" in command for command in servers))
source = (ROOT / "libs/cua-driver/rust/crates/platform-linux/src/capture.rs").read_text()
helper = source.split("impl XvfbServer {", 1)[1].split("fn stop(", 1)[0]
diff --git a/.github/workflows/ci-rust-linux.yml b/.github/workflows/ci-rust-linux.yml
index ee83def195..0e5c249cbc 100644
--- a/.github/workflows/ci-rust-linux.yml
+++ b/.github/workflows/ci-rust-linux.yml
@@ -188,8 +188,11 @@ jobs:
xvfb_primary=$!
Xvfb :99 -screen 0 1920x1080x24 -ac -noreset -nolisten tcp >xvfb-99.log 2>&1 &
xvfb_secondary=$!
- trap 'kill "$xvfb_primary" "$xvfb_secondary" 2>/dev/null || true' EXIT
- for display in :98 :99; do
+ # DirectColor is visual class 5; the usual Xvfb default is TrueColor.
+ Xvfb :100 -screen 0 1920x1080x24 -cc 5 -ac -noreset -nolisten tcp >xvfb-100.log 2>&1 &
+ xvfb_directcolor=$!
+ trap 'kill "$xvfb_primary" "$xvfb_secondary" "$xvfb_directcolor" 2>/dev/null || true' EXIT
+ for display in :98 :99 :100; do
ready=false
for _ in $(seq 1 50); do
if DISPLAY="$display" xdpyinfo >/dev/null 2>&1; then
@@ -206,15 +209,18 @@ jobs:
source_sha=$(git rev-parse HEAD)
{
echo "source_sha=$source_sha"
- DISPLAY=:98 CUA_X11_SECOND_DISPLAY=:99 CUA_X11_RESTART_DISPLAY=:100 \
+ DISPLAY=:98 CUA_X11_SECOND_DISPLAY=:99 CUA_X11_RESTART_DISPLAY=:101 \
cargo test -p platform-linux \
capture::tests::live_xshm_matches_xgetimage_across_resize_reconnect_and_repetition \
--locked -- --ignored --exact --nocapture --test-threads=1
+ DISPLAY=:100 cargo test -p platform-linux \
+ capture::tests::live_directcolor_capture_tracks_window_colormap \
+ --locked -- --ignored --exact --nocapture --test-threads=1
} 2>&1 | tee x11-capture-evidence.log
- kill "$xvfb_primary" "$xvfb_secondary" 2>/dev/null || true
- wait "$xvfb_primary" "$xvfb_secondary" || true
+ kill "$xvfb_primary" "$xvfb_secondary" "$xvfb_directcolor" 2>/dev/null || true
+ wait "$xvfb_primary" "$xvfb_secondary" "$xvfb_directcolor" || true
trap - EXIT
- sha256sum x11-capture-evidence.log xvfb-98.log xvfb-99.log >x11-capture-evidence.sha256
+ sha256sum x11-capture-evidence.log xvfb-98.log xvfb-99.log xvfb-100.log >x11-capture-evidence.sha256
- name: Upload X11 capture evidence
if: always()
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
@@ -226,6 +232,7 @@ jobs:
libs/cua-driver/rust/x11-capture-evidence.sha256
libs/cua-driver/rust/xvfb-98.log
libs/cua-driver/rust/xvfb-99.log
+ libs/cua-driver/rust/xvfb-100.log
- name: Run real uinput device-name regression
working-directory: libs/cua-driver/rust
run: |
diff --git a/docs/content/docs/reference/cua-driver/platform-support.mdx b/docs/content/docs/reference/cua-driver/platform-support.mdx
index 54eaeb0de6..5a679fbe0c 100644
--- a/docs/content/docs/reference/cua-driver/platform-support.mdx
+++ b/docs/content/docs/reference/cua-driver/platform-support.mdx
@@ -85,6 +85,11 @@ For the relationship between distributions, desktops, and compositors, read
| `cua-compositor` nested session | Experimental | Native GTK behavior, capture and scope, private route metadata, independent observation, and per-cell video | The complete shared renderer matrix is not accepted. Unicode text and a canonical parallel-drag row remain unproven; this route does not establish a stock-Wayland capability. |
| XWayland | Supported with limits | X11 routes are used when the application exposes a real X11 window; native Wayland and AT-SPI fallbacks cover mixed sessions | Capabilities depend on whether the application is actually using X11 or native Wayland. |
+X11 window capture supports packed 16-, 24-, and 32-bit TrueColor and
+DirectColor images. DirectColor capture reads the window’s current colormap
+for each frame, including palette changes that do not repaint the window.
+Other indexed visual classes remain outside the native capture path.
+
### Hyprland and Omarchy
Omarchy uses Hyprland; it is not a separate display protocol. Modern Hyprland
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/capture.rs b/libs/cua-driver/rust/crates/platform-linux/src/capture.rs
index f63cd17059..3c2ba9ed72 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/capture.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/capture.rs
@@ -146,12 +146,13 @@ struct PackedLayout {
len: usize,
}
-#[derive(Clone, Copy)]
+#[derive(Clone)]
struct PixelDecoder {
layout: PackedLayout,
red_mask: u32,
green_mask: u32,
blue_mask: u32,
+ direct_colors: Option<Vec<[u8; 3]>>,
}
impl PixelCatalog {
@@ -212,14 +213,24 @@ impl PixelCatalog {
})
}
- fn decoder(&self, w: u32, h: u32, depth: u8, visual_id: u32) -> Result<PixelDecoder> {
+ fn decoder(
+ &self,
+ w: u32,
+ h: u32,
+ depth: u8,
+ visual_id: u32,
+ query_colormap: impl FnOnce(&Visualtype) -> Result<Vec<[u8; 3]>>,
+ ) -> Result<PixelDecoder> {
let layout = self.packed_layout(w, h, depth)?;
let visual = self
.visuals
.iter()
.find(|visual| visual.visual_id == visual_id)
.ok_or_else(|| anyhow!("unknown X11 visual 0x{visual_id:x} for depth {depth}"))?;
- if visual.class != VisualClass::TRUE_COLOR {
+ if !matches!(
+ visual.class,
+ VisualClass::TRUE_COLOR | VisualClass::DIRECT_COLOR
+ ) {
bail!(
"unsupported X11 visual class {:?} for visual 0x{visual_id:x}",
visual.class
@@ -238,15 +249,65 @@ impl PixelCatalog {
{
bail!("overlapping RGB masks for X11 visual 0x{visual_id:x}");
}
+ let direct_colors = if visual.class == VisualClass::DIRECT_COLOR {
+ Some(query_colormap(visual)?)
+ } else {
+ None
+ };
Ok(PixelDecoder {
layout,
red_mask: visual.red_mask,
green_mask: visual.green_mask,
blue_mask: visual.blue_mask,
+ direct_colors,
})
}
}
+/// DirectColor masks address separate color ramps, not RGB intensities. Fetch
+/// them from the window on every frame: applications can replace the colormap
+/// or change its entries without changing the pixels or the visual.
+fn query_direct_colors(
+ conn: &x11rb::rust_connection::RustConnection,
+ window: u32,
+ visual: &Visualtype,
+) -> Result<Vec<[u8; 3]>> {
+ use x11rb::protocol::xproto::ConnectionExt as _;
+
+ let masks = [visual.red_mask, visual.green_mask, visual.blue_mask];
+ let entries = masks
+ .iter()
+ .map(|mask| (mask >> mask.trailing_zeros()) + 1)
+ .max()
+ .unwrap_or(0);
+ if entries == 0 || entries > u32::from(visual.colormap_entries) {
+ bail!("DirectColor masks exceed the visual's colormap entries");
+ }
+ let attributes = conn.get_window_attributes(window)?.reply()?;
+ if attributes.visual != visual.visual_id {
+ bail!("X11 capture visual differs from window visual");
+ }
+ let mut colors = Vec::with_capacity(entries as usize);
+ // Fit ordinary X11 request limits even for 16-bit component maps.
+ for start in (0..entries).step_by(1024) {
+ let pixels: Vec<_> = (start..entries.min(start + 1024))
+ .map(|index| {
+ masks.iter().fold(0, |pixel, mask| {
+ pixel | ((index << mask.trailing_zeros()) & mask)
+ })
+ })
+ .collect();
+ let reply = conn.query_colors(attributes.colormap, &pixels)?.reply()?;
+ if reply.colors.len() != pixels.len() {
+ bail!("X11 QueryColors returned an incomplete DirectColor map");
+ }
+ colors.extend(reply.colors.iter().map(|color| {
+ [color.red, color.green, color.blue].map(|value| ((u32::from(value) + 128) / 257) as u8)
+ }));
+ }
+ Ok(colors)
+}
+
fn checked_capture_dimensions(w: u32, h: u32) -> Result<()> {
if w == 0 || h == 0 {
bail!("window geometry is 0x0");
@@ -305,12 +366,17 @@ fn packed_zpixmap_to_png(data: &[u8], w: u32, h: u32, decoder: PixelDecoder) ->
} else {
bail!("unsupported X11 image byte order");
};
- rgba.extend_from_slice(&[
- component_to_u8(pixel, decoder.red_mask),
- component_to_u8(pixel, decoder.green_mask),
- component_to_u8(pixel, decoder.blue_mask),
- 255,
- ]);
+ let masks = [decoder.red_mask, decoder.green_mask, decoder.blue_mask];
+ for (channel, mask) in masks.into_iter().enumerate() {
+ let value = match &decoder.direct_colors {
+ Some(colors) => {
+ colors[((pixel & mask) >> mask.trailing_zeros()) as usize][channel]
+ }
+ None => component_to_u8(pixel, mask),
+ };
+ rgba.push(value);
+ }
+ rgba.push(255);
}
}
cua_driver_core::image_utils::encode_rgba_to_png(&rgba, w, h)
@@ -624,7 +690,11 @@ impl XShmSession {
data,
w,
h,
- decoder: self.pixels.decoder(w, h, reply.depth, reply.visual)?,
+ decoder: self
+ .pixels
+ .decoder(w, h, reply.depth, reply.visual, |visual| {
+ query_direct_colors(&self.conn, window, visual)
+ })?,
})
}
}
@@ -850,7 +920,9 @@ impl XGetImageSession {
data: img.data,
w,
h,
- decoder: self.pixels.decoder(w, h, img.depth, img.visual)?,
+ decoder: self.pixels.decoder(w, h, img.depth, img.visual, |visual| {
+ query_direct_colors(&self.conn, window, visual)
+ })?,
})
}
}
@@ -1051,6 +1123,7 @@ mod tests {
red_mask: 0x00ff_0000,
green_mask: 0x0000_ff00,
blue_mask: 0x0000_00ff,
+ direct_colors: None,
};
// Width 3 at 24bpp with 32-bit scanline padding: 9 payload bytes +
// 3 ignored padding bytes per row. Colors are deliberately varied.
@@ -1080,6 +1153,7 @@ mod tests {
red_mask: 0xf800,
green_mask: 0x07e0,
blue_mask: 0x001f,
+ direct_colors: None,
};
let packed = [0xf8, 0x00, 0x07, 0xe0];
let png = packed_zpixmap_to_png(&packed, 2, 1, decoder).expect("decode big-endian");
@@ -1106,19 +1180,56 @@ mod tests {
}],
visuals: vec![valid],
};
- let decoder = catalog.decoder(3, 2, 24, 0x21).expect("valid decoder");
+ let decoder = catalog
+ .decoder(3, 2, 24, 0x21, |_| {
+ panic!("TrueColor does not query a colormap")
+ })
+ .expect("valid decoder");
assert_eq!(decoder.layout.bytes_per_pixel, 3);
assert_eq!(decoder.layout.stride, 12);
assert_eq!(decoder.layout.len, 24);
catalog.visuals[0].red_mask = 0x00f0_f000;
let err = catalog
- .decoder(3, 2, 24, 0x21)
+ .decoder(3, 2, 24, 0x21, |_| {
+ panic!("TrueColor does not query a colormap")
+ })
.err()
.expect("non-contiguous mask must fail");
assert!(format!("{err:#}").contains("not contiguous"));
}
+ #[test]
+ fn directcolor_decoder_uses_independent_rgb565_indices() {
+ let catalog = PixelCatalog {
+ image_byte_order: ImageOrder::MSB_FIRST,
+ formats: vec![Format {
+ depth: 16,
+ bits_per_pixel: 16,
+ scanline_pad: 32,
+ }],
+ visuals: vec![Visualtype {
+ visual_id: 0x42,
+ class: VisualClass::DIRECT_COLOR,
+ bits_per_rgb_value: 8,
+ colormap_entries: 64,
+ red_mask: 0xf800,
+ green_mask: 0x07e0,
+ blue_mask: 0x001f,
+ }],
+ };
+ let decoder = catalog
+ .decoder(1, 1, 16, 0x42, |_| {
+ Ok((0..64)
+ .map(|index| [100 + index, 200 - index, index * 2])
+ .collect())
+ })
+ .expect("DirectColor decoder");
+ let png = packed_zpixmap_to_png(&[0x08, 0x43, 0xde, 0xad], 1, 1, decoder)
+ .expect("decode independently indexed channels and skip padding");
+ assert_eq!(decode_png_rgba(&png), vec![101, 198, 6, 255]);
+ }
+
#[test]
fn xid_narrowing_rejects_values_above_x11_range() {
assert_eq!(xid_to_window(u64::from(u32::MAX)).unwrap(), u32::MAX);
@@ -1494,6 +1605,121 @@ mod tests {
.expect("fixture sync reply");
}
+ /// DirectColor pixels index independently mutable red, green and blue maps.
+ /// A non-linear map and a map-only update distinguish correct capture from
+ /// treating the pixel masks as TrueColor intensities or caching the palette.
+ #[test]
+ #[ignore = "requires a live X11 server with DirectColor and MIT-SHM 1.2"]
+ fn live_directcolor_capture_tracks_window_colormap() {
+ use x11rb::protocol::xproto::{
+ ColorFlag, Coloritem, ColormapAlloc, ConnectionExt as _, CreateWindowAux, WindowClass,
+ };
+ use x11rb::rust_connection::RustConnection;
+
+ let display = std::env::var("DISPLAY").expect("DISPLAY must be set");
+ let (conn, screen_num) = RustConnection::connect(Some(&display)).expect("connect fixture");
+ let screen = &conn.setup().roots[screen_num];
+ let visual = screen
+ .allowed_depths
+ .iter()
+ .filter(|depth| depth.depth == 24)
+ .flat_map(|depth| &depth.visuals)
+ .find(|visual| {
+ visual.class == VisualClass::DIRECT_COLOR
+ && visual.red_mask == 0xff0000
+ && visual.green_mask == 0xff00
+ && visual.blue_mask == 0xff
+ })
+ .expect("24-bit DirectColor visual");
+ let colormap = conn.generate_id().expect("colormap id");
+ conn.create_colormap(ColormapAlloc::ALL, colormap, screen.root, visual.visual_id)
+ .unwrap()
+ .check()
+ .unwrap();
+ let window = conn.generate_id().expect("window id");
+ conn.create_window(
+ 24,
+ window,
+ screen.root,
+ 0,
+ 0,
+ 67,
+ 43,
+ 0,
+ WindowClass::INPUT_OUTPUT,
+ visual.visual_id,
+ &CreateWindowAux::new()
+ .background_pixel(0x175ba8)
+ .border_pixel(0)
+ .colormap(colormap),
+ )
+ .unwrap()
+ .check()
+ .unwrap();
+ conn.map_window(window).unwrap().check().unwrap();
+ let fixture = LiveFixture { conn, window };
+ paint_live_fixture(&fixture, 67, 43);
+ let mut shm = XShmSession::connect(display.clone()).expect("connect SHM capture");
+ let mut xget = XGetImageSession::connect(display).expect("connect XGetImage capture");
+ for phase in 0..2u16 {
+ let colors: Vec<_> = (0..256u16)
+ .map(|index| Coloritem {
+ pixel: u32::from(index) * 0x010101,
+ red: ((index + 73 + phase * 19) % 256) * 257,
+ green: (255 - index) * 257,
+ blue: ((index * 3 + phase * 31) % 256) * 257,
+ flags: ColorFlag::RED | ColorFlag::GREEN | ColorFlag::BLUE,
+ })
+ .collect();
+ fixture
+ .conn
+ .store_colors(colormap, &colors)
+ .unwrap()
+ .check()
+ .unwrap();
+ for frame in [
+ shm.capture_raw(u64::from(window))
+ .expect("DirectColor SHM capture"),
+ xget.capture_raw(u64::from(window))
+ .expect("DirectColor XGetImage capture"),
+ ] {
+ let png = raw_frame_png(frame);
+ assert_eq!(
+ cua_driver_core::image_utils::png_dimensions(&png).unwrap(),
+ (67, 43)
+ );
+ let rgba = decode_png_rgba(&png);
+ for (x, indices) in [
+ (0, [0x17u16, 0x5b, 0xa8]),
+ (33, [0xc4, 0x3d, 0x52]),
+ (66, [0x2d, 0xb8, 0x71]),
+ ] {
+ assert_eq!(
+ &rgba[x * 4..x * 4 + 4],
+ &[
+ ((indices[0] + 73 + phase * 19) % 256) as u8,
+ (255 - indices[1]) as u8,
+ ((indices[2] * 3 + phase * 31) % 256) as u8,
+ 255,
+ ]
+ );
+ }
+ }
+ }
+ fixture
+ .conn
+ .destroy_window(window)
+ .unwrap()
+ .check()
+ .unwrap();
+ fixture
+ .conn
+ .free_colormap(colormap)
+ .unwrap()
+ .check()
+ .unwrap();
+ }
+
fn raw_frame_png(frame: RawFrame) -> Vec<u8> {
packed_zpixmap_to_png(&frame.data, frame.w, frame.h, frame.decoder)
.expect("encode raw frame")