YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(e2e): stop a local suite run drawing over the developer's screen
Louis Beaumontcbac3ff9fcfb
diff --git a/apps/screenpipe-app-tauri/src-tauri/src/window/mod.rs b/apps/screenpipe-app-tauri/src-tauri/src/window/mod.rsindex b4d66e5f7..9cb74ba65 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/window/mod.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/window/mod.rs@@ -54,17 +54,38 @@ pub fn window_activation_allowed() -> bool {Ok("1") | Ok("true"));let on_ci = std::env::var("CI").is_ok_and(|v| !v.is_empty() && v != "0" && v != "false");- E2E_ACTIVATION_ALLOWED.store(opted_in || on_ci, Ordering::SeqCst);+ let allowed = opted_in || on_ci;+ E2E_ACTIVATION_ALLOWED.store(allowed, Ordering::SeqCst);+ publish_non_intrusive_env(allowed);});E2E_ACTIVATION_ALLOWED.load(Ordering::SeqCst)}+/// Carry the switch across the Rust/Swift boundary.+///+/// The macOS shortcut overlay is a set of AppKit panels owned by+/// `swift/shortcut_reminder.swift`, which this crate never gets a handle to.+/// They are the loudest thing in a suite run — floating level plus+/// `orderFrontRegardless()` is designed to cover the frontmost app without+/// taking focus, so gating activation here did nothing to them. The process+/// environment is the one channel both sides already share.+#[cfg(feature = "e2e")]+fn publish_non_intrusive_env(activation_allowed: bool) {+ std::env::set_var(+ "SCREENPIPE_E2E_NON_INTRUSIVE",+ if activation_allowed { "0" } else { "1" },+ );+}+/// Let a spec re-enable real activation for the assertions that need it.#[cfg(feature = "e2e")]pub fn set_window_activation_allowed(allowed: bool) {// Run the env seed first so it cannot clobber this later.let _ = window_activation_allowed();E2E_ACTIVATION_ALLOWED.store(allowed, std::sync::atomic::Ordering::SeqCst);+ // Keep Swift's view of the switch in step, so a spec that opts real+ // activation back in also gets the real overlay placement.+ publish_non_intrusive_env(allowed);}#[cfg(not(feature = "e2e"))]@@ -104,10 +125,12 @@ pub(crate) trait GatedPanelPlacement {#[cfg(target_os = "macos")]impl GatedPanelPlacement for tauri_nspanel::raw_nspanel::RawNSPanel {fn set_level_gated(&self, level: i32) {+ // Not `0`: a panel clamped to the normal level still orders to the front+ // of that level and covers the developer's app. Below normal it cannot.self.set_level(if window_activation_allowed() {level} else {- 0+ NON_INTRUSIVE_WINDOW_LEVEL});}@@ -156,6 +179,52 @@ pub(crate) fn activate_app_if_allowed() {}}+/// One step below `NSNormalWindowLevel`, where every real app window lives.+#[cfg(target_os = "macos")]+pub(crate) const NON_INTRUSIVE_WINDOW_LEVEL: i32 = -1;++/// Sink an e2e window below every real app window.+///+/// The gates above stop screenpipe *taking focus*, but focus is only half of+/// what puts a window over the developer's work. A window that is merely shown+/// still orders to the front of its own level, so it lands on top of whatever+/// they are looking at while focus stays exactly where it was. Every gate here+/// passed and the suite still covered the screen.+///+/// Level is the structural half of the rule. Below `NSNormalWindowLevel` a+/// window cannot be composited above any app window, whatever calls+/// `orderFront:` afterwards, so this holds without a race and without having to+/// find every show path. The window is still created, still rendered, and still+/// driven by WebDriver: it just lives underneath the developer's desktop.+#[cfg(target_os = "macos")]+pub(crate) fn sink_below_apps_if_non_intrusive(window: &tauri::WebviewWindow) {+ if window_activation_allowed() {+ return;+ }+ // AppKit window state is main-thread only, and `finalize_webview_window` runs+ // on whichever thread created the window — often a tokio worker. Reaching+ // `setLevel:` from there traps inside WindowManagement and takes the process+ // with it, so hop to the main thread before touching the NSWindow.+ let target = window.clone();+ let _ = window.run_on_main_thread(move || {+ use objc::{msg_send, sel, sel_impl};+ use tauri_nspanel::cocoa::base::{id, nil};+ let Ok(ns_window) = target.ns_window() else {+ return;+ };+ unsafe {+ let ns_window = ns_window as id;+ let _: () = msg_send![ns_window, setLevel: NON_INTRUSIVE_WINDOW_LEVEL];+ let _: () = msg_send![ns_window, orderBack: nil];+ }+ });+}++/// No-op off macOS: the non-intrusive rule exists for a developer's own desktop,+/// and local suite runs there are macOS. CI is exempt on every platform.+#[cfg(not(target_os = "macos"))]+pub(crate) fn sink_below_apps_if_non_intrusive(_window: &tauri::WebviewWindow) {}+/// Give the foreground back the instant anything makes the app active.////// The per-callsite gates above stop screenpipe *asking* for the foreground, but@@ -229,7 +298,14 @@ pub(crate) mod ns {/// # Safety/// `ns_win` must be a live `NSWindow` and the caller must be on the main thread.pub(crate) unsafe fn set_level_gated(ns_win: id, level: i64) {- let level = if window_activation_allowed() { level } else { 0 };+ // Not `0`: the normal level is where the developer's own app windows+ // live, so a window clamped there still covers them the moment anything+ // orders it front. Below normal it cannot.+ let level = if window_activation_allowed() {+ level+ } else {+ super::NON_INTRUSIVE_WINDOW_LEVEL as i64+ };let _: () = msg_send![ns_win, setLevel: level];}@@ -262,8 +338,10 @@ pub(crate) mod ns {objc::runtime::Object,>()];} else {- let _: () =- msg_send![ns_win, orderFront: std::ptr::null::<objc::runtime::Object>()];+ // `orderFront:` still lifts the window to the top of its level, which+ // is the other half of covering the developer's screen even with the+ // key-focus step skipped. Put it behind instead.+ let _: () = msg_send![ns_win, orderBack: std::ptr::null::<objc::runtime::Object>()];}}}@@ -305,6 +383,9 @@ pub fn finalize_webview_window(window: tauri::WebviewWindow) -> tauri::WebviewWiif let Err(error) = capture_protection::apply_to_new_window(&window) {tracing::warn!("{error}");}+ // Every webview window in the app is built through here, so this is the one+ // place a non-intrusive e2e run can be kept off the developer's screen.+ sink_below_apps_if_non_intrusive(&window);window}diff --git a/apps/screenpipe-app-tauri/src-tauri/swift/shortcut_reminder.swift b/apps/screenpipe-app-tauri/src-tauri/swift/shortcut_reminder.swiftindex 8234d6ea5..0d9a6f87f 100644--- a/apps/screenpipe-app-tauri/src-tauri/swift/shortcut_reminder.swift+++ b/apps/screenpipe-app-tauri/src-tauri/swift/shortcut_reminder.swift@@ -6,6 +6,49 @@ import Foundationimport AppKitimport SwiftUI+// MARK: - Non-intrusive e2e placement+//+// A local `--features e2e` run drives this overlay on the developer's own+// desktop. Every panel below is built to sit over whatever is frontmost:+// floating level, `canJoinAllSpaces`, and `orderFrontRegardless()` together put+// it on top of another app *without ever taking focus*. That is exactly right+// for the product and exactly wrong for a test run, and it is why suppressing+// activation on the Rust side never stopped the suite covering the screen.+//+// Rust owns the switch (`window_activation_allowed`) and exports it through the+// process environment, since these panels are AppKit windows this crate never+// sees. Read with `getenv` rather than a cached `ProcessInfo` snapshot: a spec+// can opt real placement back in mid-run, so the flag has to be read live.+private var nonIntrusiveE2E: Bool {+ guard let raw = getenv("SCREENPIPE_E2E_NON_INTRUSIVE") else { return false }+ return String(cString: raw) == "1"+}++/// Below `NSNormalWindowLevel`, where a panel cannot be composited over any app+/// window whatever orders it front afterwards.+private func gatedLevel(_ level: NSWindow.Level) -> NSWindow.Level {+ nonIntrusiveE2E ? NSWindow.Level(rawValue: -1) : level+}++/// Drop the bits that carry a panel onto whichever Space the developer is on.+private func gatedBehavior(+ _ behavior: NSWindow.CollectionBehavior+) -> NSWindow.CollectionBehavior {+ guard nonIntrusiveE2E else { return behavior }+ return behavior.subtracting([.canJoinAllSpaces, .moveToActiveSpace, .fullScreenAuxiliary])+}++/// `orderFrontRegardless()` is the call that puts a panel over the frontmost app+/// from the background. Under a non-intrusive run it becomes its opposite.+private func gatedOrderFront(_ window: NSWindow?) {+ guard let window else { return }+ if nonIntrusiveE2E {+ window.orderBack(nil)+ } else {+ window.orderFrontRegardless()+ }+}+// MARK: - Callback for actions (dismiss, open window, toggle meeting)public typealias ShortcutActionCallback = @convention(c) (UnsafePointer<CChar>) -> Voidprivate var gShortcutCallback: ShortcutActionCallback?@@ -1418,7 +1461,7 @@ class ShortcutReminderController: NSObject, NSWindowDelegate {}updateContent()positionPanel()- panel?.orderFrontRegardless()+ gatedOrderFront(panel)AnimationTick.shared.setVisible(true,hasActiveSignal: false@@ -1798,8 +1841,8 @@ class ShortcutReminderController: NSObject, NSWindowDelegate {defer: false)p.isFloatingPanel = true- p.level = NSWindow.Level(rawValue: Int(CGWindowLevelForKey(.floatingWindow)) + 2)- p.collectionBehavior = [.canJoinAllSpaces, .ignoresCycle, .fullScreenAuxiliary]+ p.level = gatedLevel(NSWindow.Level(rawValue: Int(CGWindowLevelForKey(.floatingWindow)) + 2))+ p.collectionBehavior = gatedBehavior([.canJoinAllSpaces, .ignoresCycle, .fullScreenAuxiliary])p.isOpaque = falsep.backgroundColor = .clearp.hasShadow = false@@ -1915,8 +1958,8 @@ class ShortcutReminderController: NSObject, NSWindowDelegate {defer: false)disclosure.isFloatingPanel = true- disclosure.level = NSWindow.Level(rawValue: panel.level.rawValue + 1)- disclosure.collectionBehavior = [.canJoinAllSpaces, .ignoresCycle, .fullScreenAuxiliary]+ disclosure.level = gatedLevel(NSWindow.Level(rawValue: panel.level.rawValue + 1))+ disclosure.collectionBehavior = gatedBehavior([.canJoinAllSpaces, .ignoresCycle, .fullScreenAuxiliary])disclosure.isOpaque = falsedisclosure.backgroundColor = .cleardisclosure.hasShadow = false@@ -1930,7 +1973,7 @@ class ShortcutReminderController: NSObject, NSWindowDelegate {disclosure.contentView = hostingdisclosure.setContentSize(size)positionDisclosurePanel(index: index)- disclosure.orderFrontRegardless()+ gatedOrderFront(disclosure)}private func positionDisclosurePanel(index: Int? = nil) {@@ -2031,7 +2074,7 @@ class ShortcutReminderController: NSObject, NSWindowDelegate {}updateTranscriptContent()positionTranscriptPanel()- transcriptPanel?.orderFrontRegardless()+ gatedOrderFront(transcriptPanel)}private func createTranscriptPanel() {@@ -2044,8 +2087,8 @@ class ShortcutReminderController: NSObject, NSWindowDelegate {defer: false)preview.isFloatingPanel = true- preview.level = NSWindow.Level(rawValue: Int(CGWindowLevelForKey(.floatingWindow)) + 2)- preview.collectionBehavior = [.canJoinAllSpaces, .ignoresCycle, .fullScreenAuxiliary]+ preview.level = gatedLevel(NSWindow.Level(rawValue: Int(CGWindowLevelForKey(.floatingWindow)) + 2))+ preview.collectionBehavior = gatedBehavior([.canJoinAllSpaces, .ignoresCycle, .fullScreenAuxiliary])preview.isOpaque = falsepreview.backgroundColor = .clearpreview.hasShadow = false@@ -2328,7 +2371,7 @@ class ShortcutReminderController: NSObject, NSWindowDelegate {stage.alphaValue = 0// Regardless, not orderFront: the overlay is used while another app is// frontmost, and a plain orderFront does nothing from the background.- stage.orderFrontRegardless()+ gatedOrderFront(stage)NSAnimationContext.runAnimationGroup { context incontext.duration = kDragStageFadeDurcontext.timingFunction = CAMediaTimingFunction(name: .easeOut)@@ -2381,8 +2424,8 @@ class ShortcutReminderController: NSObject, NSWindowDelegate {stage.isFloatingPanel = true// One level under the pill so the thing being dragged stays on top of// the targets it is being dragged between.- stage.level = NSWindow.Level(rawValue: Int(CGWindowLevelForKey(.floatingWindow)) + 1)- stage.collectionBehavior = [.canJoinAllSpaces, .ignoresCycle, .fullScreenAuxiliary]+ stage.level = gatedLevel(NSWindow.Level(rawValue: Int(CGWindowLevelForKey(.floatingWindow)) + 1))+ stage.collectionBehavior = gatedBehavior([.canJoinAllSpaces, .ignoresCycle, .fullScreenAuxiliary])stage.isOpaque = falsestage.backgroundColor = .clearstage.hasShadow = false@@ -2510,8 +2553,8 @@ class ShortcutReminderController: NSObject, NSWindowDelegate {defer: false)toast.isFloatingPanel = true- toast.level = NSWindow.Level(rawValue: Int(CGWindowLevelForKey(.floatingWindow)) + 2)- toast.collectionBehavior = [.canJoinAllSpaces, .ignoresCycle, .fullScreenAuxiliary]+ toast.level = gatedLevel(NSWindow.Level(rawValue: Int(CGWindowLevelForKey(.floatingWindow)) + 2))+ toast.collectionBehavior = gatedBehavior([.canJoinAllSpaces, .ignoresCycle, .fullScreenAuxiliary])toast.isOpaque = falsetoast.backgroundColor = .cleartoast.hasShadow = false@@ -2591,7 +2634,7 @@ class ShortcutReminderController: NSObject, NSWindowDelegate {let destination = toast.frameguard let pill = panelFrameIfVisible() else {toast.alphaValue = 1- toast.orderFrontRegardless()+ gatedOrderFront(toast)return}let start = NSRect(@@ -2602,7 +2645,7 @@ class ShortcutReminderController: NSObject, NSWindowDelegate {)toast.setFrame(start, display: false)toast.alphaValue = 0- toast.orderFrontRegardless()+ gatedOrderFront(toast)NSAnimationContext.runAnimationGroup { context incontext.duration = kAnimDurcontext.timingFunction = CAMediaTimingFunction(name: .easeOut)