Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(windows overlay): stop the pill moving out from under the pointer

louis030195b296bd021886
diff --git a/crates/screenpipe-overlay-win/src/layout.rs b/crates/screenpipe-overlay-win/src/layout.rs
index 68f01aff5..f02f17e08 100644
--- a/crates/screenpipe-overlay-win/src/layout.rs
+++ b/crates/screenpipe-overlay-win/src/layout.rs
@@ -137,21 +137,57 @@ impl Layout {
/// True when the point is over any painted surface — what decides whether
/// the window keeps the pointer or lets it fall through to the desktop.
+ ///
+ /// Every block the renderer draws has to be listed. The disclosure row was
+ /// missing, so resting the pointer on the shortcut hint read as "off the
+ /// pill": the dock collapsed, which took the hint away, which put the
+ /// pointer back on the chip — the pill flickered and its hit zone felt a
+ /// fraction of its size.
pub fn is_opaque_at(&self, px: f32, py: f32) -> bool {
- let surfaces = [Some(self.primary), self.transcript, self.notification]
- .into_iter()
- .flatten();
- for r in surfaces {
- // A couple of DIP of slack keeps the hover from flickering on the
- // 1px border, which is where the pointer naturally lands.
- if r.inset(-2.0, -2.0).contains(px, py) {
- return true;
- }
- }
- false
+ [
+ Some(self.primary),
+ self.disclosure,
+ self.transcript,
+ self.notification,
+ ]
+ .into_iter()
+ .flatten()
+ .any(|r| with_slack(r).contains(px, py))
+ }
+
+ /// True when the point is anywhere inside the overlay's own box, painted or
+ /// not — the corridor the pointer travels along to reach an attachment.
+ ///
+ /// The blocks are separated by transparent gaps, and a layered window is
+ /// not sent mouse messages over a transparent pixel: crossing from the pill
+ /// to the card arrives as `WM_MOUSELEAVE`. Collapsing there would take the
+ /// card away mid-reach, so an exit is only believed once the pointer is out
+ /// of this rect. macOS debounces the same crossing in `scheduleHoverExit`.
+ pub fn is_in_corridor(&self, px: f32, py: f32) -> bool {
+ self.window.contains(px, py)
+ }
+
+ /// True when the point is over the notification row.
+ ///
+ /// Reading a toast must neither open nor close the dock. Expanding restacks
+ /// everything between the pill and the toast — the disclosure row, the
+ /// transcript card — which slides the toast out from under the pointer, so
+ /// the click that follows lands on whatever took its place: a dock cell,
+ /// usually the timeline. macOS gives the toast its own panel, where
+ /// hovering it *holds* the pill's expansion rather than driving it; this is
+ /// the same rule expressed in one window.
+ pub fn is_notification_at(&self, px: f32, py: f32) -> bool {
+ self.notification
+ .is_some_and(|r| with_slack(r).contains(px, py))
}
}
+/// A couple of DIP of slack keeps the hover from flickering on the 1px border,
+/// which is where the pointer naturally lands.
+fn with_slack(r: Rect) -> Rect {
+ r.inset(-2.0, -2.0)
+}
+
/// Width of the primary surface for the current state.
fn primary_size(state: &OverlayState, s: f32) -> (f32, f32) {
let expanded = state.is_expanded();
@@ -526,6 +562,88 @@ mod tests {
}
}
+ #[test]
+ fn expanding_under_the_pointer_slides_the_toast_off_its_own_button() {
+ // Why the pointer must not expand the pill from the toast: hovering
+ // inserts the dock's extra height, the disclosure row, and the
+ // transcript card between the pill and the toast, so the "open note"
+ // button jumps ~190pt and a dock cell — the timeline — lands where the
+ // pointer already is. This asserts the displacement so the guard in
+ // `is_notification_at` cannot be dropped as redundant.
+ let mut s = OverlayState {
+ anchor: Anchor::BottomRight,
+ meeting_active: true,
+ transcript: vec![TranscriptItem {
+ speaker: "louis".into(),
+ text: "shipping the native overlay".into(),
+ device_type: "input".into(),
+ }],
+ notification: parse(
+ r#"{"id":"m","title":"meeting detected","body":"google meet","actions":[
+ {"label":"open note","action":"deeplink","primary":true},
+ {"label":"open note + HD","action":"api"}]}"#,
+ )
+ .ok(),
+ ..Default::default()
+ };
+
+ let resting = compute(&s);
+ let toast = resting.notification.expect("toast");
+ let (control, button) = notification_action_rects(&s, toast, resting.scale)[0];
+ assert_eq!(
+ control,
+ Control::NotificationAction0,
+ "primary is 'open note'"
+ );
+ let (px, py) = (button.x + button.w / 2.0, button.y + button.h / 2.0);
+ assert_eq!(resting.hit_test(px, py), Some(Control::NotificationAction0));
+ assert!(resting.is_notification_at(px, py));
+
+ s.hovering = true;
+ let expanded = compute(&s);
+ // The pill hugs the bottom edge, so the window grows the other way and
+ // a pointer standing still on screen sits this much further down the
+ // client area than it did.
+ let grew = expanded.window.h - resting.window.h;
+ assert!(
+ grew > 100.0,
+ "dock, disclosure and card all wedge in: {grew}"
+ );
+ assert_ne!(
+ expanded.hit_test(px, py + grew),
+ Some(Control::NotificationAction0),
+ "the button the pointer was aiming at moved out from under it"
+ );
+ assert!(
+ !expanded.is_notification_at(px, py + grew),
+ "toast moved away"
+ );
+ }
+
+ #[test]
+ fn the_toast_is_the_only_surface_that_freezes_the_dock() {
+ // The pill and the transcript still drive expansion — only the toast
+ // holds it, so the rule cannot silently swallow the pill's own hover.
+ let mut s = hovered();
+ s.meeting_active = true;
+ s.transcript_pinned = true;
+ s.transcript = vec![TranscriptItem {
+ speaker: "louis".into(),
+ text: "hello".into(),
+ device_type: "input".into(),
+ }];
+ s.notification = parse(r#"{"id":"m","title":"meeting started"}"#).ok();
+ let l = compute(&s);
+ let p = l.primary;
+ assert!(!l.is_notification_at(p.x + p.w / 2.0, p.y + p.h / 2.0));
+ let t = l.transcript.expect("transcript");
+ assert!(!l.is_notification_at(t.x + t.w / 2.0, t.y + t.h / 2.0));
+ assert!(
+ !l.is_notification_at(0.5, 0.5),
+ "shadow ring is not the toast"
+ );
+ }
+
#[test]
fn the_notification_row_hugs_its_content() {
// Two 14pt text rows, or a 22pt action button, is everything the row
@@ -652,4 +770,65 @@ mod tests {
assert!(!l.is_opaque_at(0.5, 0.5), "shadow padding is click-through");
assert!(l.is_opaque_at(l.primary.x + 1.0, l.primary.y + 1.0));
}
+
+ #[test]
+ fn every_painted_block_counts_as_the_pill() {
+ // A block the renderer draws but this does not know about reads as "the
+ // pointer left the pill": the dock collapses, which takes that block
+ // away, which flickers the whole overlay. The disclosure row shipped
+ // exactly that way.
+ let mut s = hovered();
+ s.meeting_active = true;
+ s.transcript = vec![TranscriptItem {
+ speaker: "louis".into(),
+ text: "hello".into(),
+ device_type: "input".into(),
+ }];
+ s.notification = parse(r#"{"id":"m","title":"meeting started"}"#).ok();
+ let l = compute(&s);
+ for (name, r) in [
+ ("primary", Some(l.primary)),
+ ("disclosure", l.disclosure),
+ ("transcript", l.transcript),
+ ("notification", l.notification),
+ ] {
+ let r = r.unwrap_or_else(|| panic!("{name} should be laid out"));
+ assert!(
+ l.is_opaque_at(r.x + r.w / 2.0, r.y + r.h / 2.0),
+ "{name} is painted but does not hold the hover"
+ );
+ }
+ }
+
+ #[test]
+ fn the_gaps_between_blocks_are_a_corridor_not_an_exit() {
+ // Reaching the card's buttons means crossing 4pt of transparency, which
+ // a layered window reports as the pointer leaving. Believing that would
+ // close the card mid-reach, so the gap has to stay inside the overlay.
+ let mut s = hovered();
+ s.meeting_active = true;
+ s.transcript = vec![TranscriptItem {
+ speaker: "louis".into(),
+ text: "hello".into(),
+ device_type: "input".into(),
+ }];
+ let l = compute(&s);
+ let card = l.transcript.expect("card");
+ let disclosure = l.disclosure.expect("disclosure");
+ // Bottom anchors stack the card above the disclosure, so the gap is
+ // between the two.
+ let gap_y = (card.bottom() + disclosure.y) / 2.0;
+ let gap_x = l.primary.x + l.primary.w / 2.0;
+ assert!(
+ l.is_in_corridor(gap_x, gap_y),
+ "the gap is still inside the overlay, so hover survives the reach"
+ );
+ // The shadow ring is the honest case: fully transparent, so win32 sends
+ // no message there at all and the pointer's own position is the only
+ // way to tell a reach from an exit.
+ assert!(!l.is_opaque_at(0.5, 0.5), "shadow ring is not painted");
+ assert!(l.is_in_corridor(0.5, 0.5), "but it is not an exit either");
+ assert!(!l.is_in_corridor(-1.0, gap_y), "outside the box is an exit");
+ assert!(!l.is_in_corridor(gap_x, l.window.bottom() + 1.0));
+ }
}
diff --git a/crates/screenpipe-overlay-win/src/preview.rs b/crates/screenpipe-overlay-win/src/preview.rs
index 1482c4872..eedbd62d8 100644
--- a/crates/screenpipe-overlay-win/src/preview.rs
+++ b/crates/screenpipe-overlay-win/src/preview.rs
@@ -301,13 +301,42 @@ fn parse_anchor(s: &str) -> Option<Anchor> {
})
}
+/// The started-meeting toast, minus its auto-dismiss so it stays up long enough
+/// to be aimed at. Raised through `show_notification` because `Cmd::Update`
+/// deliberately does not carry a notification — which is why the deepest stack
+/// the pill has (dock, disclosure, card, toast) could not be clicked through in
+/// the preview at all until now.
+const MEETING_TOAST: &str = r#"{
+ "id": "meeting-started-42",
+ "title": "meeting detected",
+ "body": "google meet — product sync",
+ "actions": [
+ {"label": "open note + HD", "action": "record-hd"},
+ {"label": "open note", "action": "deeplink",
+ "url": "screenpipe://meeting/42", "primary": true}
+ ]
+}"#;
+
/// Drive the real window through the flows so the interaction — not just the
/// pixels — gets exercised, then leave it up and interactive for a human.
-fn live(anchor: Anchor, cycle: bool) {
+fn live(anchor: Anchor, cycle: bool, toast: bool) {
let overlay = Overlay::spawn(|action| println!("action: {action}"));
overlay.show();
- let resting = OverlayState { anchor, ..base() };
+ let resting = if toast {
+ // The deepest stack the pill has, and the only one where every reach is
+ // exercised: chip → disclosure → live card → toast, with the card
+ // unpinned so it exists only while the pointer is on its way to it.
+ OverlayState {
+ anchor,
+ meeting_active: true,
+ meeting_id: Some(42),
+ transcript: meeting_lines(),
+ ..base()
+ }
+ } else {
+ OverlayState { anchor, ..base() }
+ };
if cycle {
let script: Vec<(u64, OverlayState)> = flows()
@@ -333,6 +362,17 @@ fn live(anchor: Anchor, cycle: bool) {
println!("resting pill is up at {anchor:?} — hover it, click it, drag it. ctrl-c to quit");
overlay.update(resting.clone());
+ if toast {
+ // `show` is a message to the overlay thread and the toast is refused
+ // until it lands, so wait for the window instead of racing it.
+ while !overlay.is_visible() {
+ std::thread::sleep(Duration::from_millis(20));
+ }
+ match overlay.show_notification(MEETING_TOAST) {
+ Ok(()) => println!("toast raised — aim at 'open note' and click"),
+ Err(reason) => eprintln!("toast refused: {reason}"),
+ }
+ }
// Feed a speech-shaped level the way the engine would. Real speech arrives
// in bursts with gaps; holding one number would tell us nothing about
@@ -374,12 +414,13 @@ pub fn run() -> windows::core::Result<()> {
.and_then(|a| parse_anchor(a))
.unwrap_or(Anchor::BottomCenter);
let cycle = !args.iter().any(|a| a == "--no-cycle");
- live(anchor, cycle);
+ let toast = args.iter().any(|a| a == "--toast");
+ live(anchor, cycle, toast);
Ok(())
}
_ => {
eprintln!(
- "usage: overlay-preview shots [dir]\n overlay-preview live-shots [dir]\n overlay-preview live [anchor] [--no-cycle]\n\nanchors: top-left top-center top-right middle-left middle-right\n bottom-left bottom-center bottom-right"
+ "usage: overlay-preview shots [dir]\n overlay-preview live-shots [dir]\n overlay-preview live [anchor] [--no-cycle] [--toast]\n\nanchors: top-left top-center top-right middle-left middle-right\n bottom-left bottom-center bottom-right"
);
Ok(())
}
diff --git a/crates/screenpipe-overlay-win/src/window.rs b/crates/screenpipe-overlay-win/src/window.rs
index f3aaf2fdc..df9e3b8a9 100644
--- a/crates/screenpipe-overlay-win/src/window.rs
+++ b/crates/screenpipe-overlay-win/src/window.rs
@@ -69,7 +69,13 @@ const WM_OVERLAY_CMD: u32 = WM_APP + 1;
const WM_OVERLAY_REPAINT: u32 = WM_APP + 2;
const ANIM_TIMER: usize = 1;
const DISMISS_TIMER: usize = 2;
+/// Runs only while the pointer is off every painted surface but still inside
+/// the overlay — see `Layout::is_in_corridor`.
+const HOVER_TIMER: usize = 3;
const ANIM_MS: u32 = 83; // ~12 Hz, same cadence as the macOS meter.
+/// How often the corridor is rechecked. Short enough that leaving the pill
+/// still feels immediate, long enough to cost nothing.
+const HOVER_POLL_MS: u32 = 90;
/// Pointer travel before a press on the pill becomes a drag rather than a click.
const DRAG_THRESHOLD: f32 = 4.0;
@@ -208,6 +214,10 @@ struct Ctx {
/// `DraggableHostingView` contract. Kept out of `OverlayState` because no
/// caller and no painter needs it.
press_can_drag: bool,
+ /// The pointer is on the notification row. Kept out of `OverlayState`
+ /// because it must not expand the pill — it only suspends the toast's
+ /// self-dismiss and freezes the expansion the pointer arrived with.
+ toast_hovered: bool,
dragging: bool,
drag_offset: (i32, i32),
animating: bool,
@@ -273,6 +283,7 @@ fn run_message_loop(
work_area: monitor_work_area_at_cursor(),
press_origin: None,
press_can_drag: false,
+ toast_hovered: false,
dragging: false,
drag_offset: (0, 0),
animating: false,
@@ -566,6 +577,11 @@ fn arm_dismiss_timer(hwnd: HWND) {
unsafe {
let Some(ctx) = ctx_of(hwnd) else { return };
let _ = KillTimer(hwnd, DISMISS_TIMER);
+ // A toast under the pointer is being read. Letting it expire mid-aim
+ // would take the click with it, so the countdown restarts on exit.
+ if ctx.toast_hovered {
+ return;
+ }
if let Some(ms) = ctx
.state
.notification
@@ -577,6 +593,39 @@ fn arm_dismiss_timer(hwnd: HWND) {
}
}
+/// The cursor in client DIP, or `None` when it is outside the window box.
+/// The overlay is a borderless popup, so the client origin is the window
+/// origin and no `ScreenToClient` round-trip is needed.
+fn cursor_in_client(hwnd: HWND) -> Option<(f32, f32)> {
+ unsafe {
+ let mut pt = POINT::default();
+ GetCursorPos(&mut pt).ok()?;
+ let mut rc = RECT::default();
+ GetWindowRect(hwnd, &mut rc).ok()?;
+ let scale = dpi_scale(hwnd);
+ Some((
+ (pt.x - rc.left) as f32 / scale,
+ (pt.y - rc.top) as f32 / scale,
+ ))
+ }
+}
+
+/// Let the pill close: the pointer really is gone.
+fn end_hover(hwnd: HWND) {
+ unsafe {
+ let _ = KillTimer(hwnd, HOVER_TIMER);
+ let Some(ctx) = ctx_of(hwnd) else { return };
+ ctx.state.hovering = false;
+ ctx.state.hovered_control = None;
+ ctx.state.pressed_control = None;
+ let was_reading = std::mem::take(&mut ctx.toast_hovered);
+ apply_state(hwnd);
+ if was_reading {
+ arm_dismiss_timer(hwnd);
+ }
+ }
+}
+
/// Client-area point in DIP for a mouse message.
fn mouse_dip(hwnd: HWND, lparam: LPARAM) -> (f32, f32) {
let x = (lparam.0 & 0xFFFF) as i16 as f32;
@@ -764,14 +813,28 @@ extern "system" fn wndproc(hwnd: HWND, msg: u32, wparam: WPARAM, lparam: LPARAM)
}
// A layered window only receives mouse messages over non-
- // transparent pixels, so anything that arrives here is a hover.
+ // transparent pixels, so anything that arrives here is a hover
+ // and the corridor no longer has to be watched.
+ let _ = KillTimer(hwnd, HOVER_TIMER);
let hovered = ctx.layout.hit_test(x, y);
let inside = ctx.layout.is_opaque_at(x, y);
- if ctx.state.hovering != inside || ctx.state.hovered_control != hovered {
- ctx.state.hovering = inside;
+ // Reading the toast holds the pill's expansion instead of
+ // driving it — see `Layout::is_notification_at`. Toggling the
+ // dock here would restack the pill and slide the toast out from
+ // under the pointer mid-aim.
+ let on_toast = ctx.layout.is_notification_at(x, y);
+ let hovering = if on_toast { ctx.state.hovering } else { inside };
+ if ctx.state.hovering != hovering || ctx.state.hovered_control != hovered {
+ ctx.state.hovering = hovering;
ctx.state.hovered_control = hovered;
apply_state(hwnd);
}
+ // …and it holds the toast open too: a notification that expires
+ // while it is being read takes the click with it.
+ if ctx.toast_hovered != on_toast {
+ ctx.toast_hovered = on_toast;
+ arm_dismiss_timer(hwnd);
+ }
let mut track = TRACKMOUSEEVENT {
cbSize: std::mem::size_of::<TRACKMOUSEEVENT>() as u32,
dwFlags: TME_LEAVE,
@@ -782,13 +845,13 @@ extern "system" fn wndproc(hwnd: HWND, msg: u32, wparam: WPARAM, lparam: LPARAM)
LRESULT(0)
}
+ // Not believed on its own: reaching for the card or the toast
+ // crosses a transparent gap, and that arrives here exactly like a
+ // real exit. The corridor timer decides which one it was.
WM_MOUSELEAVE => {
if let Some(ctx) = ctx_of(hwnd) {
if !ctx.dragging {
- ctx.state.hovering = false;
- ctx.state.hovered_control = None;
- ctx.state.pressed_control = None;
- apply_state(hwnd);
+ SetTimer(hwnd, HOVER_TIMER, HOVER_POLL_MS, None);
}
}
LRESULT(0)
@@ -869,23 +932,33 @@ extern "system" fn wndproc(hwnd: HWND, msg: u32, wparam: WPARAM, lparam: LPARAM)
update_animation_timer(hwnd);
}
}
+ HOVER_TIMER => {
+ // Still inside the overlay's own box: the pointer is
+ // between blocks on its way to one, not gone. Only the
+ // highlight follows it across the gap.
+ let inside = cursor_in_client(hwnd).filter(|(x, y)| {
+ ctx_of(hwnd).is_some_and(|c| c.layout.is_in_corridor(*x, *y))
+ });
+ match inside {
+ Some((x, y)) => {
+ if let Some(ctx) = ctx_of(hwnd) {
+ let hovered = ctx.layout.hit_test(x, y);
+ if ctx.state.hovered_control != hovered {
+ ctx.state.hovered_control = hovered;
+ repaint(hwnd);
+ }
+ }
+ }
+ None => end_hover(hwnd),
+ }
+ }
DISMISS_TIMER => {
let _ = KillTimer(hwnd, DISMISS_TIMER);
if let Some(ctx) = ctx_of(hwnd) {
- // Leave an alert the pointer is resting on: the user
- // is reading it, and its buttons are under the mouse.
- let engaged = ctx.state.hovering
- && matches!(
- ctx.state.hovered_control,
- Some(
- Control::NotificationAction0
- | Control::NotificationAction1
- | Control::NotificationDismiss
- )
- );
- if engaged {
- arm_dismiss_timer(hwnd);
- } else if ctx.state.notification.take().is_some() {
+ // A toast under the pointer is being read;
+ // `arm_dismiss_timer` restarts the countdown when
+ // the pointer leaves it.
+ if !ctx.toast_hovered && ctx.state.notification.take().is_some() {
apply_state(hwnd);
}
}