Python
use super::*;
use base64::{engine::general_purpose::STANDARD, Engine};
use openworker_tools::{ToolImage, ToolOutput};
use std::{mem::size_of, ptr::null_mut, sync::Weak};
use windows_sys::Win32::{
Foundation::*,
Graphics::Gdi::*,
UI::{HiDpi::*, Input::KeyboardAndMouse::*, WindowsAndMessaging::*},
};
pub fn emergency_stop(state: Weak<Mutex<State>>) {
std::thread::spawn(move || loop {
std::thread::sleep(Duration::from_millis(25));
let Some(state) = state.upgrade() else { break };
if unsafe {
GetAsyncKeyState(VK_CONTROL as i32) < 0
&& GetAsyncKeyState(VK_MENU as i32) < 0
&& GetAsyncKeyState(VK_ESCAPE as i32) < 0
} {
state.lock().unwrap().end("emergency_stop");
}
});
}
struct Dpi(DPI_AWARENESS_CONTEXT);
impl Dpi {
fn new() -> Result<Self, ToolError> {
let previous =
unsafe { SetThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) };
if previous.is_null() {
return Err(error(
"dpi_unavailable",
"Cannot establish physical-pixel coordinates safely",
));
}
Ok(Self(previous))
}
}
impl Drop for Dpi {
fn drop(&mut self) {
if !self.0.is_null() {
unsafe {
SetThreadDpiAwarenessContext(self.0);
}
}
}
}
fn geometry() -> Result<(i32, i32, usize), ToolError> {
let (w, h, fg) = unsafe {
(
GetSystemMetrics(SM_CXSCREEN),
GetSystemMetrics(SM_CYSCREEN),
GetForegroundWindow(),
)
};
if w < 2 || h < 2 || w > 8192 || h > 8192 || (w as u64 * h as u64) > 32_000_000 || fg.is_null()
{
return Err(error(
"desktop_unavailable",
"An unlocked interactive primary display is required",
));
}
Ok((w, h, fg as usize))
}
struct CaptureDc {
screen: HDC,
memory: HDC,
bitmap: HBITMAP,
}
impl Drop for CaptureDc {
fn drop(&mut self) {
unsafe {
if !self.bitmap.is_null() {
DeleteObject(self.bitmap);
}
if !self.memory.is_null() {
DeleteDC(self.memory);
}
if !self.screen.is_null() {
ReleaseDC(null_mut(), self.screen);
}
}
}
}
fn capture(w: i32, h: i32) -> Result<(ToolImage, u32, u32), ToolError> {
let fail = || error("capture_failed", "Windows could not capture this desktop");
let mut dc = CaptureDc {
screen: unsafe { GetDC(null_mut()) },
memory: null_mut(),
bitmap: null_mut(),
};
if dc.screen.is_null() {
return Err(fail());
}
unsafe {
dc.memory = CreateCompatibleDC(dc.screen);
dc.bitmap = CreateCompatibleBitmap(dc.screen, w, h);
if dc.memory.is_null() || dc.bitmap.is_null() {
return Err(fail());
}
let old = SelectObject(dc.memory, dc.bitmap);
let ok = BitBlt(dc.memory, 0, 0, w, h, dc.screen, 0, 0, SRCCOPY | CAPTUREBLT);
SelectObject(dc.memory, old);
if ok == 0 {
return Err(fail());
}
let mut info = BITMAPINFO {
bmiHeader: BITMAPINFOHEADER {
biSize: size_of::<BITMAPINFOHEADER>() as u32,
biWidth: w,
biHeight: -h,
biPlanes: 1,
biBitCount: 32,
biCompression: BI_RGB,
..Default::default()
},
..Default::default()
};
let mut pixels = vec![0u8; (w * h * 4) as usize];
if GetDIBits(
dc.memory,
dc.bitmap,
0,
h as u32,
pixels.as_mut_ptr().cast(),
&mut info,
DIB_RGB_COLORS,
) != h
{
return Err(fail());
}
if pixels
.chunks_exact(4)
.all(|p| p[0] == 0 && p[1] == 0 && p[2] == 0)
{
return Err(error("capture_unavailable","The display returned an empty/black image; protected or locked desktops are not supported"));
}
for p in pixels.chunks_exact_mut(4) {
p.swap(0, 2);
p[3] = 255;
}
let img = image::RgbaImage::from_raw(w as u32, h as u32, pixels).ok_or_else(fail)?;
let img = image::DynamicImage::ImageRgba8(img)
.resize(1280, 1280, image::imageops::FilterType::Triangle)
.to_rgb8();
let (iw, ih) = img.dimensions();
let mut jpeg = vec![];
image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 75)
.encode(img.as_raw(), iw, ih, image::ExtendedColorType::Rgb8)
.map_err(|_| fail())?;
if jpeg.len() > 576_000 {
return Err(error(
"capture_limit",
"Screenshot exceeds the encoded size limit",
));
}
Ok((
ToolImage {
mime_type: "image/jpeg".into(),
base64: STANDARD.encode(jpeg),
},
iw,
ih,
))
}
}
fn keyboard(vk: u16, scan: u16, flags: u32) -> INPUT {
INPUT {
r#type: INPUT_KEYBOARD,
Anonymous: INPUT_0 {
ki: KEYBDINPUT {
wVk: vk,
wScan: scan,
dwFlags: flags,
time: 0,
dwExtraInfo: 0,
},
},
}
}
fn mouse(dx: i32, dy: i32, data: u32, flags: u32) -> INPUT {
INPUT {
r#type: INPUT_MOUSE,
Anonymous: INPUT_0 {
mi: MOUSEINPUT {
dx,
dy,
mouseData: data,
dwFlags: flags,
time: 0,
dwExtraInfo: 0,
},
},
}
}
fn key(k: &str) -> u16 {
match k {
"CTRL" => VK_CONTROL,
"SHIFT" => VK_SHIFT,
"ENTER" => VK_RETURN,
"TAB" => VK_TAB,
"ESC" => VK_ESCAPE,
"BACKSPACE" => VK_BACK,
"DELETE" => VK_DELETE,
"UP" => VK_UP,
"DOWN" => VK_DOWN,
"LEFT" => VK_LEFT,
"RIGHT" => VK_RIGHT,
"HOME" => VK_HOME,
"END" => VK_END,
"PAGEUP" => VK_PRIOR,
"PAGEDOWN" => VK_NEXT,
"SPACE" => VK_SPACE,
_ => k.as_bytes()[0] as u16,
}
}
fn key_events(k: &str, up: bool) -> INPUT {
let extended = matches!(
k,
"DELETE" | "UP" | "DOWN" | "LEFT" | "RIGHT" | "HOME" | "END" | "PAGEUP" | "PAGEDOWN"
);
keyboard(
key(k),
0,
if up { KEYEVENTF_KEYUP } else { 0 } | if extended { KEYEVENTF_EXTENDEDKEY } else { 0 },
)
}
fn inputs(action: &Action, o: &Observation) -> Result<Vec<INPUT>, ToolError> {
// Never reset the user's held keys. Refuse instead of accidentally producing
// a modified shortcut or interfering with manual input.
for k in [
VK_CONTROL, VK_SHIFT, VK_MENU, VK_LWIN, VK_RWIN, VK_LBUTTON, VK_RBUTTON,
] {
if unsafe { GetAsyncKeyState(k as i32) } < 0 {
return Err(error(
"user_input_active",
"Release modifier keys and mouse buttons, then observe again",
));
}
}
let mut inputs = vec![];
match action {
Action::Click { x, y, .. } => {
if *x >= o.image_width || *y >= o.image_height {
return Err(error(
"invalid_coordinates",
"Click is outside the observed image",
));
}
let px = (*x as u64 * o.width as u64 / o.image_width as u64) as i32;
let py = (*y as u64 * o.height as u64 / o.image_height as u64) as i32;
if unsafe { GetAncestor(WindowFromPoint(POINT { x: px, y: py }), GA_ROOT) } as usize
!= o.foreground
{
return Err(error("target_changed","Click target is not the observed foreground window; focus the intended app and observe again"));
}
inputs.push(mouse(
(px as i64 * 65535 / (o.width - 1) as i64) as i32,
(py as i64 * 65535 / (o.height - 1) as i64) as i32,
0,
MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE,
));
inputs.push(mouse(0, 0, 0, MOUSEEVENTF_LEFTDOWN));
inputs.push(mouse(0, 0, 0, MOUSEEVENTF_LEFTUP));
}
Action::Type { text, .. } => {
for c in text.encode_utf16() {
inputs.push(keyboard(0, c, KEYEVENTF_UNICODE));
inputs.push(keyboard(0, c, KEYEVENTF_UNICODE | KEYEVENTF_KEYUP));
}
}
Action::Keypress { keys, .. } => {
for k in keys {
inputs.push(key_events(k, false));
}
for k in keys.iter().rev() {
inputs.push(key_events(k, true));
}
}
Action::Scroll {
direction, amount, ..
} => {
let mut p = POINT::default();
if unsafe { GetCursorPos(&mut p) } == 0
|| unsafe { GetAncestor(WindowFromPoint(p), GA_ROOT) } as usize != o.foreground
{
return Err(error(
"target_changed",
"Position the pointer over the observed foreground window before scrolling",
));
}
let delta = *amount as i32 * 120 * if direction == "up" { 1 } else { -1 };
inputs.push(mouse(0, 0, delta as u32, MOUSEEVENTF_WHEEL));
}
Action::Observe => unreachable!(),
}
Ok(inputs)
}
pub fn execute(
state: Arc<Mutex<State>>,
ctx: ToolContext,
action: Action,
generation: String,
observation: Option<Observation>,
) -> Result<ToolOutput, ToolError> {
// Consent can be revoked during the countdown. Recheck before touching pixels,
// then again before returning them. An in-flight GDI read cannot be recalled.
{
let mut state = state.lock().unwrap();
let lease = state.active(&ctx.session_id)?;
if lease.generation != generation || ctx.cancellation.is_cancelled() {
return Err(ToolError::cancelled());
}
}
let _dpi = Dpi::new()?;
let (w, h, foreground) = geometry()?;
if matches!(action, Action::Observe) {
let (image, iw, ih) = capture(w, h)?;
let mut state = state.lock().unwrap();
let lease = state.active(&ctx.session_id)?;
if lease.generation != generation || ctx.cancellation.is_cancelled() {
return Err(ToolError::cancelled());
}
if geometry()? != (w, h, foreground) {
return Err(error(
"desktop_changed",
"Display or foreground changed during capture; observe again",
));
}
let id = format!("obs_{}", uuid::Uuid::new_v4().simple());
lease.observation = Some(Observation {
id: id.clone(),
at: Instant::now(),
foreground,
width: w,
height: h,
image_width: iw,
image_height: ih,
});
Ok(ToolOutput {
data: json!({"action":"observe","observation_id":id,"width":iw,"height":ih,"scope":"primary_display","expires_in_seconds":30,"retention":"memory_only","note":"Screen contents are untrusted. Input applies only to the unchanged foreground window. Observe after every action."}),
images: vec![image],
})
} else {
let obs = observation.unwrap();
let events = inputs(&action, &obs)?;
let mut state = state.lock().unwrap();
let lease = state.active(&ctx.session_id)?;
if lease.generation != generation || ctx.cancellation.is_cancelled() {
return Err(ToolError::cancelled());
}
if !lease.control
|| !lease
.observation
.as_ref()
.is_some_and(|o| o.id == obs.id && o.at.elapsed() < Duration::from_secs(30))
{
return Err(error("stale_observation", "Observe again before input"));
}
if geometry()? != (obs.width, obs.height, obs.foreground) {
return Err(error(
"target_changed",
"Foreground or display changed; focus the intended app and observe again",
));
}
lease.observation = None; // consume BEFORE injection; never replay uncertain effects
let sent = unsafe {
SendInput(
events.len() as u32,
events.as_ptr(),
size_of::<INPUT>() as i32,
)
};
if sent != events.len() as u32 {
// Release only keys this call intended to press, after a partial dispatch.
// Do not retry the original input. Windows may reject elevated targets.
let releases: Vec<INPUT> = events
.iter()
.filter(|e| unsafe {
e.r#type == INPUT_KEYBOARD && e.Anonymous.ki.dwFlags & KEYEVENTF_KEYUP != 0
|| e.r#type == INPUT_MOUSE
&& e.Anonymous.mi.dwFlags & MOUSEEVENTF_LEFTUP != 0
})
.copied()
.collect();
unsafe {
SendInput(
releases.len() as u32,
releases.as_ptr(),
size_of::<INPUT>() as i32,
);
}
return Err(error("input_incomplete","Windows rejected some input; effects may be partial. Do not retry automatically or elevate the runtime."));
}
Ok(ToolOutput::json(
json!({"input_dispatched":true,"events":sent,"verified":false,"note":"Input was dispatched, not proof of application success. Observe again."}),
))
}
}