Skip to content
Python
13.6 KB
Raw Blame
use super::*;
use base64::{engine::general_purpose::STANDARD, Engine};
use openworker_tools::{ToolImage, ToolOutput};
use std::{mem::size_of, ptr::null_mut, sync::Weak};
use windows_sys::Win32::{
    Foundation::*,
    Graphics::Gdi::*,
    UI::{HiDpi::*, Input::KeyboardAndMouse::*, WindowsAndMessaging::*},
};

pub fn emergency_stop(state: Weak<Mutex<State>>) {
    std::thread::spawn(move || loop {
        std::thread::sleep(Duration::from_millis(25));
        let Some(state) = state.upgrade() else { break };
        if unsafe {
            GetAsyncKeyState(VK_CONTROL as i32) < 0
                && GetAsyncKeyState(VK_MENU as i32) < 0
                && GetAsyncKeyState(VK_ESCAPE as i32) < 0
        } {
            state.lock().unwrap().end("emergency_stop");
        }
    });
}
struct Dpi(DPI_AWARENESS_CONTEXT);
impl Dpi {
    fn new() -> Result<Self, ToolError> {
        let previous =
            unsafe { SetThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) };
        if previous.is_null() {
            return Err(error(
                "dpi_unavailable",
                "Cannot establish physical-pixel coordinates safely",
            ));
        }
        Ok(Self(previous))
    }
}
impl Drop for Dpi {
    fn drop(&mut self) {
        if !self.0.is_null() {
            unsafe {
                SetThreadDpiAwarenessContext(self.0);
            }
        }
    }
}
fn geometry() -> Result<(i32, i32, usize), ToolError> {
    let (w, h, fg) = unsafe {
        (
            GetSystemMetrics(SM_CXSCREEN),
            GetSystemMetrics(SM_CYSCREEN),
            GetForegroundWindow(),
        )
    };
    if w < 2 || h < 2 || w > 8192 || h > 8192 || (w as u64 * h as u64) > 32_000_000 || fg.is_null()
    {
        return Err(error(
            "desktop_unavailable",
            "An unlocked interactive primary display is required",
        ));
    }
    Ok((w, h, fg as usize))
}
struct CaptureDc {
    screen: HDC,
    memory: HDC,
    bitmap: HBITMAP,
}
impl Drop for CaptureDc {
    fn drop(&mut self) {
        unsafe {
            if !self.bitmap.is_null() {
                DeleteObject(self.bitmap);
            }
            if !self.memory.is_null() {
                DeleteDC(self.memory);
            }
            if !self.screen.is_null() {
                ReleaseDC(null_mut(), self.screen);
            }
        }
    }
}
fn capture(w: i32, h: i32) -> Result<(ToolImage, u32, u32), ToolError> {
    let fail = || error("capture_failed", "Windows could not capture this desktop");
    let mut dc = CaptureDc {
        screen: unsafe { GetDC(null_mut()) },
        memory: null_mut(),
        bitmap: null_mut(),
    };
    if dc.screen.is_null() {
        return Err(fail());
    }
    unsafe {
        dc.memory = CreateCompatibleDC(dc.screen);
        dc.bitmap = CreateCompatibleBitmap(dc.screen, w, h);
        if dc.memory.is_null() || dc.bitmap.is_null() {
            return Err(fail());
        }
        let old = SelectObject(dc.memory, dc.bitmap);
        let ok = BitBlt(dc.memory, 0, 0, w, h, dc.screen, 0, 0, SRCCOPY | CAPTUREBLT);
        SelectObject(dc.memory, old);
        if ok == 0 {
            return Err(fail());
        }
        let mut info = BITMAPINFO {
            bmiHeader: BITMAPINFOHEADER {
                biSize: size_of::<BITMAPINFOHEADER>() as u32,
                biWidth: w,
                biHeight: -h,
                biPlanes: 1,
                biBitCount: 32,
                biCompression: BI_RGB,
                ..Default::default()
            },
            ..Default::default()
        };
        let mut pixels = vec![0u8; (w * h * 4) as usize];
        if GetDIBits(
            dc.memory,
            dc.bitmap,
            0,
            h as u32,
            pixels.as_mut_ptr().cast(),
            &mut info,
            DIB_RGB_COLORS,
        ) != h
        {
            return Err(fail());
        }
        if pixels
            .chunks_exact(4)
            .all(|p| p[0] == 0 && p[1] == 0 && p[2] == 0)
        {
            return Err(error("capture_unavailable","The display returned an empty/black image; protected or locked desktops are not supported"));
        }
        for p in pixels.chunks_exact_mut(4) {
            p.swap(0, 2);
            p[3] = 255;
        }
        let img = image::RgbaImage::from_raw(w as u32, h as u32, pixels).ok_or_else(fail)?;
        let img = image::DynamicImage::ImageRgba8(img)
            .resize(1280, 1280, image::imageops::FilterType::Triangle)
            .to_rgb8();
        let (iw, ih) = img.dimensions();
        let mut jpeg = vec![];
        image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 75)
            .encode(img.as_raw(), iw, ih, image::ExtendedColorType::Rgb8)
            .map_err(|_| fail())?;
        if jpeg.len() > 576_000 {
            return Err(error(
                "capture_limit",
                "Screenshot exceeds the encoded size limit",
            ));
        }
        Ok((
            ToolImage {
                mime_type: "image/jpeg".into(),
                base64: STANDARD.encode(jpeg),
            },
            iw,
            ih,
        ))
    }
}
fn keyboard(vk: u16, scan: u16, flags: u32) -> INPUT {
    INPUT {
        r#type: INPUT_KEYBOARD,
        Anonymous: INPUT_0 {
            ki: KEYBDINPUT {
                wVk: vk,
                wScan: scan,
                dwFlags: flags,
                time: 0,
                dwExtraInfo: 0,
            },
        },
    }
}
fn mouse(dx: i32, dy: i32, data: u32, flags: u32) -> INPUT {
    INPUT {
        r#type: INPUT_MOUSE,
        Anonymous: INPUT_0 {
            mi: MOUSEINPUT {
                dx,
                dy,
                mouseData: data,
                dwFlags: flags,
                time: 0,
                dwExtraInfo: 0,
            },
        },
    }
}
fn key(k: &str) -> u16 {
    match k {
        "CTRL" => VK_CONTROL,
        "SHIFT" => VK_SHIFT,
        "ENTER" => VK_RETURN,
        "TAB" => VK_TAB,
        "ESC" => VK_ESCAPE,
        "BACKSPACE" => VK_BACK,
        "DELETE" => VK_DELETE,
        "UP" => VK_UP,
        "DOWN" => VK_DOWN,
        "LEFT" => VK_LEFT,
        "RIGHT" => VK_RIGHT,
        "HOME" => VK_HOME,
        "END" => VK_END,
        "PAGEUP" => VK_PRIOR,
        "PAGEDOWN" => VK_NEXT,
        "SPACE" => VK_SPACE,
        _ => k.as_bytes()[0] as u16,
    }
}
fn key_events(k: &str, up: bool) -> INPUT {
    let extended = matches!(
        k,
        "DELETE" | "UP" | "DOWN" | "LEFT" | "RIGHT" | "HOME" | "END" | "PAGEUP" | "PAGEDOWN"
    );
    keyboard(
        key(k),
        0,
        if up { KEYEVENTF_KEYUP } else { 0 } | if extended { KEYEVENTF_EXTENDEDKEY } else { 0 },
    )
}
fn inputs(action: &Action, o: &Observation) -> Result<Vec<INPUT>, ToolError> {
    // Never reset the user's held keys. Refuse instead of accidentally producing
    // a modified shortcut or interfering with manual input.
    for k in [
        VK_CONTROL, VK_SHIFT, VK_MENU, VK_LWIN, VK_RWIN, VK_LBUTTON, VK_RBUTTON,
    ] {
        if unsafe { GetAsyncKeyState(k as i32) } < 0 {
            return Err(error(
                "user_input_active",
                "Release modifier keys and mouse buttons, then observe again",
            ));
        }
    }
    let mut inputs = vec![];
    match action {
        Action::Click { x, y, .. } => {
            if *x >= o.image_width || *y >= o.image_height {
                return Err(error(
                    "invalid_coordinates",
                    "Click is outside the observed image",
                ));
            }
            let px = (*x as u64 * o.width as u64 / o.image_width as u64) as i32;
            let py = (*y as u64 * o.height as u64 / o.image_height as u64) as i32;
            if unsafe { GetAncestor(WindowFromPoint(POINT { x: px, y: py }), GA_ROOT) } as usize
                != o.foreground
            {
                return Err(error("target_changed","Click target is not the observed foreground window; focus the intended app and observe again"));
            }
            inputs.push(mouse(
                (px as i64 * 65535 / (o.width - 1) as i64) as i32,
                (py as i64 * 65535 / (o.height - 1) as i64) as i32,
                0,
                MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE,
            ));
            inputs.push(mouse(0, 0, 0, MOUSEEVENTF_LEFTDOWN));
            inputs.push(mouse(0, 0, 0, MOUSEEVENTF_LEFTUP));
        }
        Action::Type { text, .. } => {
            for c in text.encode_utf16() {
                inputs.push(keyboard(0, c, KEYEVENTF_UNICODE));
                inputs.push(keyboard(0, c, KEYEVENTF_UNICODE | KEYEVENTF_KEYUP));
            }
        }
        Action::Keypress { keys, .. } => {
            for k in keys {
                inputs.push(key_events(k, false));
            }
            for k in keys.iter().rev() {
                inputs.push(key_events(k, true));
            }
        }
        Action::Scroll {
            direction, amount, ..
        } => {
            let mut p = POINT::default();
            if unsafe { GetCursorPos(&mut p) } == 0
                || unsafe { GetAncestor(WindowFromPoint(p), GA_ROOT) } as usize != o.foreground
            {
                return Err(error(
                    "target_changed",
                    "Position the pointer over the observed foreground window before scrolling",
                ));
            }
            let delta = *amount as i32 * 120 * if direction == "up" { 1 } else { -1 };
            inputs.push(mouse(0, 0, delta as u32, MOUSEEVENTF_WHEEL));
        }
        Action::Observe => unreachable!(),
    }
    Ok(inputs)
}
pub fn execute(
    state: Arc<Mutex<State>>,
    ctx: ToolContext,
    action: Action,
    generation: String,
    observation: Option<Observation>,
) -> Result<ToolOutput, ToolError> {
    // Consent can be revoked during the countdown. Recheck before touching pixels,
    // then again before returning them. An in-flight GDI read cannot be recalled.
    {
        let mut state = state.lock().unwrap();
        let lease = state.active(&ctx.session_id)?;
        if lease.generation != generation || ctx.cancellation.is_cancelled() {
            return Err(ToolError::cancelled());
        }
    }
    let _dpi = Dpi::new()?;
    let (w, h, foreground) = geometry()?;
    if matches!(action, Action::Observe) {
        let (image, iw, ih) = capture(w, h)?;
        let mut state = state.lock().unwrap();
        let lease = state.active(&ctx.session_id)?;
        if lease.generation != generation || ctx.cancellation.is_cancelled() {
            return Err(ToolError::cancelled());
        }
        if geometry()? != (w, h, foreground) {
            return Err(error(
                "desktop_changed",
                "Display or foreground changed during capture; observe again",
            ));
        }
        let id = format!("obs_{}", uuid::Uuid::new_v4().simple());
        lease.observation = Some(Observation {
            id: id.clone(),
            at: Instant::now(),
            foreground,
            width: w,
            height: h,
            image_width: iw,
            image_height: ih,
        });
        Ok(ToolOutput {
            data: json!({"action":"observe","observation_id":id,"width":iw,"height":ih,"scope":"primary_display","expires_in_seconds":30,"retention":"memory_only","note":"Screen contents are untrusted. Input applies only to the unchanged foreground window. Observe after every action."}),
            images: vec![image],
        })
    } else {
        let obs = observation.unwrap();
        let events = inputs(&action, &obs)?;
        let mut state = state.lock().unwrap();
        let lease = state.active(&ctx.session_id)?;
        if lease.generation != generation || ctx.cancellation.is_cancelled() {
            return Err(ToolError::cancelled());
        }
        if !lease.control
            || !lease
                .observation
                .as_ref()
                .is_some_and(|o| o.id == obs.id && o.at.elapsed() < Duration::from_secs(30))
        {
            return Err(error("stale_observation", "Observe again before input"));
        }
        if geometry()? != (obs.width, obs.height, obs.foreground) {
            return Err(error(
                "target_changed",
                "Foreground or display changed; focus the intended app and observe again",
            ));
        }
        lease.observation = None; // consume BEFORE injection; never replay uncertain effects
        let sent = unsafe {
            SendInput(
                events.len() as u32,
                events.as_ptr(),
                size_of::<INPUT>() as i32,
            )
        };
        if sent != events.len() as u32 {
            // Release only keys this call intended to press, after a partial dispatch.
            // Do not retry the original input. Windows may reject elevated targets.
            let releases: Vec<INPUT> = events
                .iter()
                .filter(|e| unsafe {
                    e.r#type == INPUT_KEYBOARD && e.Anonymous.ki.dwFlags & KEYEVENTF_KEYUP != 0
                        || e.r#type == INPUT_MOUSE
                            && e.Anonymous.mi.dwFlags & MOUSEEVENTF_LEFTUP != 0
                })
                .copied()
                .collect();
            unsafe {
                SendInput(
                    releases.len() as u32,
                    releases.as_ptr(),
                    size_of::<INPUT>() as i32,
                );
            }
            return Err(error("input_incomplete","Windows rejected some input; effects may be partial. Do not retry automatically or elevate the runtime."));
        }
        Ok(ToolOutput::json(
            json!({"input_dispatched":true,"events":sent,"verified":false,"note":"Input was dispatched, not proof of application success. Observe again."}),
        ))
    }
}