YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(windows overlay): stop the pill moving out from under the pointer
louis030195b296bd021886
diff --git a/crates/screenpipe-overlay-win/src/layout.rs b/crates/screenpipe-overlay-win/src/layout.rsindex 68f01aff5..f02f17e08 100644--- a/crates/screenpipe-overlay-win/src/layout.rs+++ b/crates/screenpipe-overlay-win/src/layout.rs@@ -137,21 +137,57 @@ impl Layout {/// True when the point is over any painted surface — what decides whether/// the window keeps the pointer or lets it fall through to the desktop.+ ///+ /// Every block the renderer draws has to be listed. The disclosure row was+ /// missing, so resting the pointer on the shortcut hint read as "off the+ /// pill": the dock collapsed, which took the hint away, which put the+ /// pointer back on the chip — the pill flickered and its hit zone felt a+ /// fraction of its size.pub fn is_opaque_at(&self, px: f32, py: f32) -> bool {- let surfaces = [Some(self.primary), self.transcript, self.notification]- .into_iter()- .flatten();- for r in surfaces {- // A couple of DIP of slack keeps the hover from flickering on the- // 1px border, which is where the pointer naturally lands.- if r.inset(-2.0, -2.0).contains(px, py) {- return true;- }- }- false+ [+ Some(self.primary),+ self.disclosure,+ self.transcript,+ self.notification,+ ]+ .into_iter()+ .flatten()+ .any(|r| with_slack(r).contains(px, py))+ }++ /// True when the point is anywhere inside the overlay's own box, painted or+ /// not — the corridor the pointer travels along to reach an attachment.+ ///+ /// The blocks are separated by transparent gaps, and a layered window is+ /// not sent mouse messages over a transparent pixel: crossing from the pill+ /// to the card arrives as `WM_MOUSELEAVE`. Collapsing there would take the+ /// card away mid-reach, so an exit is only believed once the pointer is out+ /// of this rect. macOS debounces the same crossing in `scheduleHoverExit`.+ pub fn is_in_corridor(&self, px: f32, py: f32) -> bool {+ self.window.contains(px, py)+ }++ /// True when the point is over the notification row.+ ///+ /// Reading a toast must neither open nor close the dock. Expanding restacks+ /// everything between the pill and the toast — the disclosure row, the+ /// transcript card — which slides the toast out from under the pointer, so+ /// the click that follows lands on whatever took its place: a dock cell,+ /// usually the timeline. macOS gives the toast its own panel, where+ /// hovering it *holds* the pill's expansion rather than driving it; this is+ /// the same rule expressed in one window.+ pub fn is_notification_at(&self, px: f32, py: f32) -> bool {+ self.notification+ .is_some_and(|r| with_slack(r).contains(px, py))}}+/// A couple of DIP of slack keeps the hover from flickering on the 1px border,+/// which is where the pointer naturally lands.+fn with_slack(r: Rect) -> Rect {+ r.inset(-2.0, -2.0)+}+/// Width of the primary surface for the current state.fn primary_size(state: &OverlayState, s: f32) -> (f32, f32) {let expanded = state.is_expanded();@@ -526,6 +562,88 @@ mod tests {}}+ #[test]+ fn expanding_under_the_pointer_slides_the_toast_off_its_own_button() {+ // Why the pointer must not expand the pill from the toast: hovering+ // inserts the dock's extra height, the disclosure row, and the+ // transcript card between the pill and the toast, so the "open note"+ // button jumps ~190pt and a dock cell — the timeline — lands where the+ // pointer already is. This asserts the displacement so the guard in+ // `is_notification_at` cannot be dropped as redundant.+ let mut s = OverlayState {+ anchor: Anchor::BottomRight,+ meeting_active: true,+ transcript: vec![TranscriptItem {+ speaker: "louis".into(),+ text: "shipping the native overlay".into(),+ device_type: "input".into(),+ }],+ notification: parse(+ r#"{"id":"m","title":"meeting detected","body":"google meet","actions":[+ {"label":"open note","action":"deeplink","primary":true},+ {"label":"open note + HD","action":"api"}]}"#,+ )+ .ok(),+ ..Default::default()+ };++ let resting = compute(&s);+ let toast = resting.notification.expect("toast");+ let (control, button) = notification_action_rects(&s, toast, resting.scale)[0];+ assert_eq!(+ control,+ Control::NotificationAction0,+ "primary is 'open note'"+ );+ let (px, py) = (button.x + button.w / 2.0, button.y + button.h / 2.0);+ assert_eq!(resting.hit_test(px, py), Some(Control::NotificationAction0));+ assert!(resting.is_notification_at(px, py));++ s.hovering = true;+ let expanded = compute(&s);+ // The pill hugs the bottom edge, so the window grows the other way and+ // a pointer standing still on screen sits this much further down the+ // client area than it did.+ let grew = expanded.window.h - resting.window.h;+ assert!(+ grew > 100.0,+ "dock, disclosure and card all wedge in: {grew}"+ );+ assert_ne!(+ expanded.hit_test(px, py + grew),+ Some(Control::NotificationAction0),+ "the button the pointer was aiming at moved out from under it"+ );+ assert!(+ !expanded.is_notification_at(px, py + grew),+ "toast moved away"+ );+ }++ #[test]+ fn the_toast_is_the_only_surface_that_freezes_the_dock() {+ // The pill and the transcript still drive expansion — only the toast+ // holds it, so the rule cannot silently swallow the pill's own hover.+ let mut s = hovered();+ s.meeting_active = true;+ s.transcript_pinned = true;+ s.transcript = vec![TranscriptItem {+ speaker: "louis".into(),+ text: "hello".into(),+ device_type: "input".into(),+ }];+ s.notification = parse(r#"{"id":"m","title":"meeting started"}"#).ok();+ let l = compute(&s);+ let p = l.primary;+ assert!(!l.is_notification_at(p.x + p.w / 2.0, p.y + p.h / 2.0));+ let t = l.transcript.expect("transcript");+ assert!(!l.is_notification_at(t.x + t.w / 2.0, t.y + t.h / 2.0));+ assert!(+ !l.is_notification_at(0.5, 0.5),+ "shadow ring is not the toast"+ );+ }+#[test]fn the_notification_row_hugs_its_content() {// Two 14pt text rows, or a 22pt action button, is everything the row@@ -652,4 +770,65 @@ mod tests {assert!(!l.is_opaque_at(0.5, 0.5), "shadow padding is click-through");assert!(l.is_opaque_at(l.primary.x + 1.0, l.primary.y + 1.0));}++ #[test]+ fn every_painted_block_counts_as_the_pill() {+ // A block the renderer draws but this does not know about reads as "the+ // pointer left the pill": the dock collapses, which takes that block+ // away, which flickers the whole overlay. The disclosure row shipped+ // exactly that way.+ let mut s = hovered();+ s.meeting_active = true;+ s.transcript = vec![TranscriptItem {+ speaker: "louis".into(),+ text: "hello".into(),+ device_type: "input".into(),+ }];+ s.notification = parse(r#"{"id":"m","title":"meeting started"}"#).ok();+ let l = compute(&s);+ for (name, r) in [+ ("primary", Some(l.primary)),+ ("disclosure", l.disclosure),+ ("transcript", l.transcript),+ ("notification", l.notification),+ ] {+ let r = r.unwrap_or_else(|| panic!("{name} should be laid out"));+ assert!(+ l.is_opaque_at(r.x + r.w / 2.0, r.y + r.h / 2.0),+ "{name} is painted but does not hold the hover"+ );+ }+ }++ #[test]+ fn the_gaps_between_blocks_are_a_corridor_not_an_exit() {+ // Reaching the card's buttons means crossing 4pt of transparency, which+ // a layered window reports as the pointer leaving. Believing that would+ // close the card mid-reach, so the gap has to stay inside the overlay.+ let mut s = hovered();+ s.meeting_active = true;+ s.transcript = vec![TranscriptItem {+ speaker: "louis".into(),+ text: "hello".into(),+ device_type: "input".into(),+ }];+ let l = compute(&s);+ let card = l.transcript.expect("card");+ let disclosure = l.disclosure.expect("disclosure");+ // Bottom anchors stack the card above the disclosure, so the gap is+ // between the two.+ let gap_y = (card.bottom() + disclosure.y) / 2.0;+ let gap_x = l.primary.x + l.primary.w / 2.0;+ assert!(+ l.is_in_corridor(gap_x, gap_y),+ "the gap is still inside the overlay, so hover survives the reach"+ );+ // The shadow ring is the honest case: fully transparent, so win32 sends+ // no message there at all and the pointer's own position is the only+ // way to tell a reach from an exit.+ assert!(!l.is_opaque_at(0.5, 0.5), "shadow ring is not painted");+ assert!(l.is_in_corridor(0.5, 0.5), "but it is not an exit either");+ assert!(!l.is_in_corridor(-1.0, gap_y), "outside the box is an exit");+ assert!(!l.is_in_corridor(gap_x, l.window.bottom() + 1.0));+ }}diff --git a/crates/screenpipe-overlay-win/src/preview.rs b/crates/screenpipe-overlay-win/src/preview.rsindex 1482c4872..eedbd62d8 100644--- a/crates/screenpipe-overlay-win/src/preview.rs+++ b/crates/screenpipe-overlay-win/src/preview.rs@@ -301,13 +301,42 @@ fn parse_anchor(s: &str) -> Option<Anchor> {})}+/// The started-meeting toast, minus its auto-dismiss so it stays up long enough+/// to be aimed at. Raised through `show_notification` because `Cmd::Update`+/// deliberately does not carry a notification — which is why the deepest stack+/// the pill has (dock, disclosure, card, toast) could not be clicked through in+/// the preview at all until now.+const MEETING_TOAST: &str = r#"{+ "id": "meeting-started-42",+ "title": "meeting detected",+ "body": "google meet — product sync",+ "actions": [+ {"label": "open note + HD", "action": "record-hd"},+ {"label": "open note", "action": "deeplink",+ "url": "screenpipe://meeting/42", "primary": true}+ ]+}"#;+/// Drive the real window through the flows so the interaction — not just the/// pixels — gets exercised, then leave it up and interactive for a human.-fn live(anchor: Anchor, cycle: bool) {+fn live(anchor: Anchor, cycle: bool, toast: bool) {let overlay = Overlay::spawn(|action| println!("action: {action}"));overlay.show();- let resting = OverlayState { anchor, ..base() };+ let resting = if toast {+ // The deepest stack the pill has, and the only one where every reach is+ // exercised: chip → disclosure → live card → toast, with the card+ // unpinned so it exists only while the pointer is on its way to it.+ OverlayState {+ anchor,+ meeting_active: true,+ meeting_id: Some(42),+ transcript: meeting_lines(),+ ..base()+ }+ } else {+ OverlayState { anchor, ..base() }+ };if cycle {let script: Vec<(u64, OverlayState)> = flows()@@ -333,6 +362,17 @@ fn live(anchor: Anchor, cycle: bool) {println!("resting pill is up at {anchor:?} — hover it, click it, drag it. ctrl-c to quit");overlay.update(resting.clone());+ if toast {+ // `show` is a message to the overlay thread and the toast is refused+ // until it lands, so wait for the window instead of racing it.+ while !overlay.is_visible() {+ std::thread::sleep(Duration::from_millis(20));+ }+ match overlay.show_notification(MEETING_TOAST) {+ Ok(()) => println!("toast raised — aim at 'open note' and click"),+ Err(reason) => eprintln!("toast refused: {reason}"),+ }+ }// Feed a speech-shaped level the way the engine would. Real speech arrives// in bursts with gaps; holding one number would tell us nothing about@@ -374,12 +414,13 @@ pub fn run() -> windows::core::Result<()> {.and_then(|a| parse_anchor(a)).unwrap_or(Anchor::BottomCenter);let cycle = !args.iter().any(|a| a == "--no-cycle");- live(anchor, cycle);+ let toast = args.iter().any(|a| a == "--toast");+ live(anchor, cycle, toast);Ok(())}_ => {eprintln!(- "usage: overlay-preview shots [dir]\n overlay-preview live-shots [dir]\n overlay-preview live [anchor] [--no-cycle]\n\nanchors: top-left top-center top-right middle-left middle-right\n bottom-left bottom-center bottom-right"+ "usage: overlay-preview shots [dir]\n overlay-preview live-shots [dir]\n overlay-preview live [anchor] [--no-cycle] [--toast]\n\nanchors: top-left top-center top-right middle-left middle-right\n bottom-left bottom-center bottom-right");Ok(())}diff --git a/crates/screenpipe-overlay-win/src/window.rs b/crates/screenpipe-overlay-win/src/window.rsindex f3aaf2fdc..df9e3b8a9 100644--- a/crates/screenpipe-overlay-win/src/window.rs+++ b/crates/screenpipe-overlay-win/src/window.rs@@ -69,7 +69,13 @@ const WM_OVERLAY_CMD: u32 = WM_APP + 1;const WM_OVERLAY_REPAINT: u32 = WM_APP + 2;const ANIM_TIMER: usize = 1;const DISMISS_TIMER: usize = 2;+/// Runs only while the pointer is off every painted surface but still inside+/// the overlay — see `Layout::is_in_corridor`.+const HOVER_TIMER: usize = 3;const ANIM_MS: u32 = 83; // ~12 Hz, same cadence as the macOS meter.+/// How often the corridor is rechecked. Short enough that leaving the pill+/// still feels immediate, long enough to cost nothing.+const HOVER_POLL_MS: u32 = 90;/// Pointer travel before a press on the pill becomes a drag rather than a click.const DRAG_THRESHOLD: f32 = 4.0;@@ -208,6 +214,10 @@ struct Ctx {/// `DraggableHostingView` contract. Kept out of `OverlayState` because no/// caller and no painter needs it.press_can_drag: bool,+ /// The pointer is on the notification row. Kept out of `OverlayState`+ /// because it must not expand the pill — it only suspends the toast's+ /// self-dismiss and freezes the expansion the pointer arrived with.+ toast_hovered: bool,dragging: bool,drag_offset: (i32, i32),animating: bool,@@ -273,6 +283,7 @@ fn run_message_loop(work_area: monitor_work_area_at_cursor(),press_origin: None,press_can_drag: false,+ toast_hovered: false,dragging: false,drag_offset: (0, 0),animating: false,@@ -566,6 +577,11 @@ fn arm_dismiss_timer(hwnd: HWND) {unsafe {let Some(ctx) = ctx_of(hwnd) else { return };let _ = KillTimer(hwnd, DISMISS_TIMER);+ // A toast under the pointer is being read. Letting it expire mid-aim+ // would take the click with it, so the countdown restarts on exit.+ if ctx.toast_hovered {+ return;+ }if let Some(ms) = ctx.state.notification@@ -577,6 +593,39 @@ fn arm_dismiss_timer(hwnd: HWND) {}}+/// The cursor in client DIP, or `None` when it is outside the window box.+/// The overlay is a borderless popup, so the client origin is the window+/// origin and no `ScreenToClient` round-trip is needed.+fn cursor_in_client(hwnd: HWND) -> Option<(f32, f32)> {+ unsafe {+ let mut pt = POINT::default();+ GetCursorPos(&mut pt).ok()?;+ let mut rc = RECT::default();+ GetWindowRect(hwnd, &mut rc).ok()?;+ let scale = dpi_scale(hwnd);+ Some((+ (pt.x - rc.left) as f32 / scale,+ (pt.y - rc.top) as f32 / scale,+ ))+ }+}++/// Let the pill close: the pointer really is gone.+fn end_hover(hwnd: HWND) {+ unsafe {+ let _ = KillTimer(hwnd, HOVER_TIMER);+ let Some(ctx) = ctx_of(hwnd) else { return };+ ctx.state.hovering = false;+ ctx.state.hovered_control = None;+ ctx.state.pressed_control = None;+ let was_reading = std::mem::take(&mut ctx.toast_hovered);+ apply_state(hwnd);+ if was_reading {+ arm_dismiss_timer(hwnd);+ }+ }+}+/// Client-area point in DIP for a mouse message.fn mouse_dip(hwnd: HWND, lparam: LPARAM) -> (f32, f32) {let x = (lparam.0 & 0xFFFF) as i16 as f32;@@ -764,14 +813,28 @@ extern "system" fn wndproc(hwnd: HWND, msg: u32, wparam: WPARAM, lparam: LPARAM)}// A layered window only receives mouse messages over non-- // transparent pixels, so anything that arrives here is a hover.+ // transparent pixels, so anything that arrives here is a hover+ // and the corridor no longer has to be watched.+ let _ = KillTimer(hwnd, HOVER_TIMER);let hovered = ctx.layout.hit_test(x, y);let inside = ctx.layout.is_opaque_at(x, y);- if ctx.state.hovering != inside || ctx.state.hovered_control != hovered {- ctx.state.hovering = inside;+ // Reading the toast holds the pill's expansion instead of+ // driving it — see `Layout::is_notification_at`. Toggling the+ // dock here would restack the pill and slide the toast out from+ // under the pointer mid-aim.+ let on_toast = ctx.layout.is_notification_at(x, y);+ let hovering = if on_toast { ctx.state.hovering } else { inside };+ if ctx.state.hovering != hovering || ctx.state.hovered_control != hovered {+ ctx.state.hovering = hovering;ctx.state.hovered_control = hovered;apply_state(hwnd);}+ // …and it holds the toast open too: a notification that expires+ // while it is being read takes the click with it.+ if ctx.toast_hovered != on_toast {+ ctx.toast_hovered = on_toast;+ arm_dismiss_timer(hwnd);+ }let mut track = TRACKMOUSEEVENT {cbSize: std::mem::size_of::<TRACKMOUSEEVENT>() as u32,dwFlags: TME_LEAVE,@@ -782,13 +845,13 @@ extern "system" fn wndproc(hwnd: HWND, msg: u32, wparam: WPARAM, lparam: LPARAM)LRESULT(0)}+ // Not believed on its own: reaching for the card or the toast+ // crosses a transparent gap, and that arrives here exactly like a+ // real exit. The corridor timer decides which one it was.WM_MOUSELEAVE => {if let Some(ctx) = ctx_of(hwnd) {if !ctx.dragging {- ctx.state.hovering = false;- ctx.state.hovered_control = None;- ctx.state.pressed_control = None;- apply_state(hwnd);+ SetTimer(hwnd, HOVER_TIMER, HOVER_POLL_MS, None);}}LRESULT(0)@@ -869,23 +932,33 @@ extern "system" fn wndproc(hwnd: HWND, msg: u32, wparam: WPARAM, lparam: LPARAM)update_animation_timer(hwnd);}}+ HOVER_TIMER => {+ // Still inside the overlay's own box: the pointer is+ // between blocks on its way to one, not gone. Only the+ // highlight follows it across the gap.+ let inside = cursor_in_client(hwnd).filter(|(x, y)| {+ ctx_of(hwnd).is_some_and(|c| c.layout.is_in_corridor(*x, *y))+ });+ match inside {+ Some((x, y)) => {+ if let Some(ctx) = ctx_of(hwnd) {+ let hovered = ctx.layout.hit_test(x, y);+ if ctx.state.hovered_control != hovered {+ ctx.state.hovered_control = hovered;+ repaint(hwnd);+ }+ }+ }+ None => end_hover(hwnd),+ }+ }DISMISS_TIMER => {let _ = KillTimer(hwnd, DISMISS_TIMER);if let Some(ctx) = ctx_of(hwnd) {- // Leave an alert the pointer is resting on: the user- // is reading it, and its buttons are under the mouse.- let engaged = ctx.state.hovering- && matches!(- ctx.state.hovered_control,- Some(- Control::NotificationAction0- | Control::NotificationAction1- | Control::NotificationDismiss- )- );- if engaged {- arm_dismiss_timer(hwnd);- } else if ctx.state.notification.take().is_some() {+ // A toast under the pointer is being read;+ // `arm_dismiss_timer` restarts the countdown when+ // the pointer leaves it.+ if !ctx.toast_hovered && ctx.state.notification.take().is_some() {apply_state(hwnd);}}