YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(live-view): route pinch to source window (#6548)
Louis Beaumontbd729ed28a46
diff --git a/COVERAGE.md b/COVERAGE.mdindex 384fa501b..ff0f2d224 100644--- a/COVERAGE.md+++ b/COVERAGE.md@@ -19,14 +19,14 @@ results and `cargo llvm-cov` data on top when judging release confidence.### Tauri E2E- Mapped specs: 119-- Declared test blocks: 345-- Weighted coverage points: 270.6+- Declared test blocks: 346+- Weighted coverage points: 271.3| Platform | Specs | Declared tests | Weighted points | Layers | Features | Critical score || --- | --- | --- | --- | --- | --- | --- || windows | 91 | 299 | 244.7 | 15 | 97 | 92% |-| macos | 115 | 307 | 240.4 | 17 | 103 | 90% |-| linux | 80 | 257 | 213.9 | 14 | 93 | 88% |+| macos | 115 | 308 | 241.1 | 17 | 104 | 90% |+| linux | 80 | 258 | 214.6 | 14 | 94 | 88% |### Core Enginediff --git a/apps/screenpipe-app-tauri/components/settings/__tests__/live-view-canvas.test.tsx b/apps/screenpipe-app-tauri/components/settings/__tests__/live-view-canvas.test.tsxindex dbbed3c1c..ad3e42da5 100644--- a/apps/screenpipe-app-tauri/components/settings/__tests__/live-view-canvas.test.tsx+++ b/apps/screenpipe-app-tauri/components/settings/__tests__/live-view-canvas.test.tsx@@ -341,6 +341,116 @@ describe("LiveViewCanvas", () => {}});+ it.each(["pane", "Block", "toolbar"])(+ "cancels proposal focus before a %s click can look like zoom",+ (targetKind) => {+ vi.stubGlobal(+ "matchMedia",+ vi.fn(() => ({ matches: false })),+ );+ vi.useFakeTimers();+ try {+ const cancelAnimationFrame = vi.fn();+ vi.stubGlobal(+ "requestAnimationFrame",+ vi.fn(() => 77),+ );+ vi.stubGlobal("cancelAnimationFrame", cancelAnimationFrame);+ const initialDocument = createCanvasDocument(view);+ const focusDocument = {+ ...initialDocument,+ blocks: initialDocument.blocks.map((block) =>+ block.slotId === "focus-time"+ ? { ...block, width: 1_800, height: 1_000 }+ : block,+ ),+ };+ const result = render(+ <CanvasHarness initialDocument={focusDocument} />,+ );++ result.rerender(+ <CanvasHarness+ initialDocument={focusDocument}+ focusSlotId="focus-time"+ />,+ );+ expect(cancelAnimationFrame).not.toHaveBeenCalledWith(77);++ const surface = screen.getByTestId("live-view-canvas-surface");+ const target =+ targetKind === "Block"+ ? screen.getByTestId("canvas-block-focus-time")+ : targetKind === "toolbar"+ ? screen.getByTestId("canvas-tools-toggle")+ : surface.querySelector<HTMLElement>(".react-flow__pane");+ expect(target).toBeTruthy();+ fireEvent.pointerDown(target!);++ expect(cancelAnimationFrame).toHaveBeenCalledWith(77);+ act(() => vi.advanceTimersByTime(360));+ openCanvasTools();+ expect(screen.getByText("100%")).toBeTruthy();+ } finally {+ vi.useRealTimers();+ }+ },+ );++ it.each(["accessibility click", "keyboard input"])(+ "cancels proposal focus before %s can look like zoom",+ (interaction) => {+ vi.stubGlobal(+ "matchMedia",+ vi.fn(() => ({ matches: false })),+ );+ vi.useFakeTimers();+ try {+ const cancelAnimationFrame = vi.fn();+ vi.stubGlobal(+ "requestAnimationFrame",+ vi.fn(() => 91),+ );+ vi.stubGlobal("cancelAnimationFrame", cancelAnimationFrame);+ const result = render(<CanvasHarness />);++ result.rerender(<CanvasHarness focusSlotId="focus-time" />);+ expect(cancelAnimationFrame).not.toHaveBeenCalledWith(91);++ if (interaction === "accessibility click") {+ // fireEvent.click has no preceding pointer event, matching keyboard+ // or assistive-technology button activation.+ fireEvent.click(screen.getByTestId("canvas-tools-toggle"));+ } else {+ fireEvent.keyDown(screen.getByTestId("live-view-canvas-surface"), {+ key: "Shift",+ });+ }++ expect(cancelAnimationFrame).toHaveBeenCalledWith(91);+ act(() => vi.advanceTimersByTime(360));+ if (interaction === "keyboard input") openCanvasTools();+ expect(screen.getByText("100%")).toBeTruthy();+ } finally {+ vi.useRealTimers();+ }+ },+ );++ it("keeps ordinary pane and Block clicks at the same zoom", () => {+ render(<CanvasHarness />);+ const surface = screen.getByTestId("live-view-canvas-surface");+ const pane = surface.querySelector<HTMLElement>(".react-flow__pane");+ expect(pane).toBeTruthy();++ fireEvent.click(pane!);+ fireEvent.doubleClick(pane!);+ fireEvent.click(screen.getByTestId("canvas-block-focus-time"));++ openCanvasTools();+ expect(screen.getByText("100%")).toBeTruthy();+ });+it("keeps the source-backed Live View cards interactive inside the canvas", () => {render(<CanvasHarness />);@@ -828,7 +938,7 @@ describe("LiveViewCanvas", () => {expect(onPersist).not.toHaveBeenCalled();});- it("ignores app-wide native pinches outside the focused canvas", async () => {+ it("accepts a window-scoped pinch without DOM focus but still requires hover", async () => {vi.spyOn(document, "hasFocus").mockReturnValue(false);render(<CanvasHarness />);const surface = screen.getByTestId("live-view-canvas-surface");@@ -841,13 +951,12 @@ describe("LiveViewCanvas", () => {act(() => {eventMocks.listeners.get("native-magnify")?.({ payload: 0.1 });});- expect(screen.getByText("100%")).toBeTruthy();+ expect(screen.getByText("110%")).toBeTruthy();- vi.mocked(document.hasFocus).mockReturnValue(true);fireEvent.pointerLeave(surface);act(() => {eventMocks.listeners.get("native-magnify")?.({ payload: 0.1 });});- expect(screen.getByText("100%")).toBeTruthy();+ expect(screen.getByText("110%")).toBeTruthy();});});diff --git a/apps/screenpipe-app-tauri/components/settings/live-view-canvas.tsx b/apps/screenpipe-app-tauri/components/settings/live-view-canvas.tsxindex a99bb62f1..64c329477 100644--- a/apps/screenpipe-app-tauri/components/settings/live-view-canvas.tsx+++ b/apps/screenpipe-app-tauri/components/settings/live-view-canvas.tsx@@ -1230,10 +1230,11 @@ export function LiveViewCanvas({}// The recognizer proves this platform delivers native pinch, so the// canvas stops trusting ctrl+wheel to describe the same fingers. Latch- // before the focus/hover guards below: a pinch this canvas ignores still- // tells us the recognizer is running.+ // before the hover guard below: a pinch outside this canvas still tells us+ // the recognizer is running. Rust scopes native events to their source+ // window, so an inactive window under the pointer remains interactive.nativeMagnifySeenRef.current = true;- if (!canvasActiveRef.current || !window.document.hasFocus()) return;+ if (!canvasActiveRef.current) return;// A real pinch takes over from an in-flight AI focus animation, the same// way a trusted pan or wheel zoom does.cancelFocusAnimation();@@ -1436,11 +1437,19 @@ export function LiveViewCanvas({TOOL_OPTIONS.find((option) => option.value === tool) ?? TOOL_OPTIONS[0];const ActiveToolIcon = activeTool.icon;+ // Any deliberate interaction owns the viewport immediately. The toolbar is+ // a sibling of the React Flow surface, so cancellation belongs on their+ // common ancestor: otherwise an in-flight AI proposal focus can keep moving+ // underneath a pointer, keyboard, or accessibility click and make that+ // interaction appear to zoom.return (<sectiondata-testid="live-view-canvas"className="relative min-h-0 w-full flex-1 overflow-hidden border border-border bg-background"aria-label="Live View process canvas"+ onPointerDownCapture={() => cancelFocusAnimation()}+ onClickCapture={() => cancelFocusAnimation()}+ onKeyDownCapture={() => cancelFocusAnimation()}><divref={surfaceRef}diff --git a/apps/screenpipe-app-tauri/e2e/COVERAGE.md b/apps/screenpipe-app-tauri/e2e/COVERAGE.mdindex 5b52d1c41..1f32fb010 100644--- a/apps/screenpipe-app-tauri/e2e/COVERAGE.md+++ b/apps/screenpipe-app-tauri/e2e/COVERAGE.md@@ -7,8 +7,8 @@ and layer declared in the manifest, weighted by confidence and criticality.- Manifest: `e2e/coverage-map.json`- Specs directory: `e2e/specs`- Mapped specs: 119-- Declared test blocks: 345-- Weighted coverage points: 270.6+- Declared test blocks: 346+- Weighted coverage points: 271.3Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -20,8 +20,8 @@ can execute more runtime cases than this number shows.| Platform | Specs | Declared tests | Weighted points | Layers | Features | Critical score || --- | --- | --- | --- | --- | --- | --- || windows | 91 | 299 | 244.7 | 15 | 97 | 92% |-| macos | 115 | 307 | 240.4 | 17 | 103 | 90% |-| linux | 80 | 257 | 213.9 | 14 | 93 | 88% |+| macos | 115 | 308 | 241.1 | 17 | 104 | 90% |+| linux | 80 | 258 | 214.6 | 14 | 94 | 88% |## Runtime Results@@ -45,10 +45,10 @@ pass/fail/skip counts.| os-integration | 7 specs / 30 tests / 25.5 pts | 14 specs / 27 tests / 16.0 pts | 2 specs / 13 tests / 9.4 pts || performance | 2 specs / 44 tests / 44.0 pts | 4 specs / 34 tests / 30.5 pts | 1 specs / 29 tests / 29.0 pts || pipes | 6 specs / 19 tests / 19.0 pts | 8 specs / 25 tests / 25.0 pts | 6 specs / 19 tests / 19.0 pts |-| real-ui-e2e | 65 specs / 200 tests / 165.4 pts | 78 specs / 204 tests / 167.6 pts | 60 specs / 176 tests / 151.2 pts |+| real-ui-e2e | 65 specs / 200 tests / 165.4 pts | 78 specs / 205 tests / 168.3 pts | 60 specs / 177 tests / 151.9 pts || settings | 14 specs / 40 tests / 37.0 pts | 16 specs / 34 tests / 29.7 pts | 13 specs / 31 tests / 28.0 pts || storage-privacy | 9 specs / 42 tests / 33.3 pts | 9 specs / 27 tests / 26.1 pts | 6 specs / 20 tests / 19.1 pts |-| tauri-command | 19 specs / 51 tests / 39.6 pts | 27 specs / 62 tests / 47.1 pts | 18 specs / 52 tests / 40.2 pts |+| tauri-command | 19 specs / 51 tests / 39.6 pts | 27 specs / 63 tests / 47.8 pts | 18 specs / 53 tests / 40.9 pts || window-lifecycle | 19 specs / 65 tests / 54.5 pts | 19 specs / 45 tests / 31.9 pts | 13 specs / 39 tests / 29.9 pts |## Critical Feature Matrix@@ -162,7 +162,7 @@ pass/fail/skip counts.| home-window.spec.ts | windows, macos, linux | real-ui-e2e, window-lifecycle | app-launch, home-navigation, timeline, settings-recording, pipes | high | strong | real-user-flow | 1 | Clicks through Home, Pipes, Timeline, Help, and Settings. || html-artifact-render.spec.ts | windows, macos, linux | real-ui-e2e | brain, artifacts, html-sandbox | high | strong | real-user-flow | 1 | Registers an HTML artifact, opens it in Brain, and asserts it renders inside a sandboxed allow-scripts iframe (CSP default-src 'none') whose global <style> never leaks into the host app DOM (regression: rehype-raw repainting the whole window). || live-view-item-actions.spec.ts | windows, macos, linux | real-ui-e2e, local-api, pipes | brain-overview, live-view-item-actions, artifacts, pipes | high | strong | real-user-flow | 1 | Installs the generic Commitments and Accounting Live View kits, shows Done, Later, and Not right without hover, persists snooze, correction, resolve, dismiss, and reopen decisions through the local API, verifies receipts survive reload, and captures real product screenshots. |-| live-view-pinch-zoom.spec.ts | macos, linux | real-ui-e2e, tauri-command | brain-overview, live-view-canvas-pinch-zoom | medium | strong | real-user-flow | 3 | Drives the Live View canvas with the same native-magnify payload the macOS magnification gesture recognizer emits. Asserts the recognizer takes ownership so WebKit's duplicate ctrl+wheel stops zooming while plain scroll still pans, that an app-wide pinch is ignored by an unfocused canvas, and that canvas zoom stays anchored, clamped at 25%, and persisted through load_brain_view_canvas. The focused per-frame zoom arithmetic is asserted in the component tests because WKWebView under WebDriver does not reliably receive OS-level focus. |+| live-view-pinch-zoom.spec.ts | macos, linux | real-ui-e2e, tauri-command | brain-overview, live-view-canvas-pinch-zoom, live-view-canvas-click-stability | medium | strong | real-user-flow | 4 | Drives the Live View canvas with the same window-scoped native-magnify payload the macOS magnification gesture recognizer emits. Asserts pinch works even when WKWebView lacks DOM focus, the recognizer takes ownership so WebKit's duplicate ctrl+wheel stops zooming while plain scroll still pans, ordinary pane, Block, and double clicks do not change the viewport, and canvas zoom stays anchored, clamped at 25%, and persisted through load_brain_view_canvas. || macos-ui-performance.spec.ts | macos | performance, real-ui-e2e | timeline, audio-device-health | medium | conditional | performance | 2 | macOS-only timeline/audio UI performance guards. || main-overlay-visibility.spec.ts | windows, macos, linux | window-lifecycle, tauri-command | window-lifecycle, main-overlay | medium | partial | command | 1 | Main overlay show/hide without duplicate handles. || main-window-close-reopen.spec.ts | windows, macos, linux | window-lifecycle, tauri-command | window-lifecycle, main-window | medium | partial | command | 1 | Main close/reopen without handle leaks. |diff --git a/apps/screenpipe-app-tauri/e2e/coverage-map.json b/apps/screenpipe-app-tauri/e2e/coverage-map.jsonindex 5e3135766..a9c817b6e 100644--- a/apps/screenpipe-app-tauri/e2e/coverage-map.json+++ b/apps/screenpipe-app-tauri/e2e/coverage-map.json@@ -1570,12 +1570,13 @@],"features": ["brain-overview",- "live-view-canvas-pinch-zoom"+ "live-view-canvas-pinch-zoom",+ "live-view-canvas-click-stability"],"criticality": "medium","confidence": "strong","ux": "real-user-flow",- "notes": "Drives the Live View canvas with the same native-magnify payload the macOS magnification gesture recognizer emits. Asserts the recognizer takes ownership so WebKit's duplicate ctrl+wheel stops zooming while plain scroll still pans, that an app-wide pinch is ignored by an unfocused canvas, and that canvas zoom stays anchored, clamped at 25%, and persisted through load_brain_view_canvas. The focused per-frame zoom arithmetic is asserted in the component tests because WKWebView under WebDriver does not reliably receive OS-level focus."+ "notes": "Drives the Live View canvas with the same window-scoped native-magnify payload the macOS magnification gesture recognizer emits. Asserts pinch works even when WKWebView lacks DOM focus, the recognizer takes ownership so WebKit's duplicate ctrl+wheel stops zooming while plain scroll still pans, ordinary pane, Block, and double clicks do not change the viewport, and canvas zoom stays anchored, clamped at 25%, and persisted through load_brain_view_canvas."},{"spec": "macos-ui-performance.spec.ts",diff --git a/apps/screenpipe-app-tauri/e2e/specs/live-view-pinch-zoom.spec.ts b/apps/screenpipe-app-tauri/e2e/specs/live-view-pinch-zoom.spec.tsindex ba53df472..b426f8e91 100644--- a/apps/screenpipe-app-tauri/e2e/specs/live-view-pinch-zoom.spec.ts+++ b/apps/screenpipe-app-tauri/e2e/specs/live-view-pinch-zoom.spec.ts@@ -16,18 +16,16 @@ import { saveScreenshot } from "../helpers/screenshot-utils.js";// same "native-magnify" payload the macOS NSMagnificationGestureRecognizer// emits.//-// Scope note: the canvas ignores a pinch unless `document.hasFocus()` is true,-// because `native-magnify` is an app-wide event that must not zoom a-// background window. WKWebView under WebDriver does not reliably receive-// OS-level focus — window-activation.spec.ts records the same limitation on-// hosted macOS runners, and `show_window_activated` does not change it. So the-// focused zoom arithmetic (1 + magnification per frame) is asserted in-// components/settings/__tests__/live-view-canvas.test.tsx, and this spec covers-// what the real app can prove without OS focus:+// Rust routes native magnification only to the WebviewWindow under the+// gesture. WKWebView under WebDriver does not reliably receive OS-level focus+// — window-activation.spec.ts records the same limitation on hosted macOS+// runners — so this spec intentionally proves that window-scoped pinch still+// works without DOM focus://-// - a pinch arriving at an unfocused canvas changes nothing+// - a pinch targeted at the canvas window works even without DOM focus// - once the recognizer has spoken, the duplicate ctrl+wheel for the same// gesture stops zooming, while plain scroll still pans+// - ordinary pane, Block, and double clicks do not change the viewport// - the canvas zoom controls stay anchored, clamped, and persistedconst VIEW_ID = "pinch-zoom-e2e";@@ -149,7 +147,8 @@ async function seedEntitledAccount(): Promise<void> {}async function emitNativeMagnify(magnification: number): Promise<void> {- await invokeOrThrow("plugin:event|emit", {+ await invokeOrThrow("plugin:event|emit_to", {+ target: { kind: "AnyLabel", label: "home" },event: "native-magnify",payload: magnification,});@@ -344,12 +343,13 @@ describe("Live View canvas pinch zoom", function () {const afterBrowserPinch = await paintedViewport();expect(afterBrowserPinch.zoom).toBeGreaterThan(before.zoom);- // The recognizer now proves it is running. Even though this canvas ignores- // the gesture itself while unfocused, it must stop trusting ctrl+wheel to- // describe the same fingers.+ // The recognizer now proves it is running. The native step applies even+ // without DOM focus because the event is scoped to this window, and the+ // canvas must stop trusting ctrl+wheel to describe the same fingers.await emitNativeMagnify(0.05);await browser.pause(t(200));const afterRecognizer = await paintedViewport();+ expect(afterRecognizer.zoom).toBeGreaterThan(afterBrowserPinch.zoom);await dispatchWheel(anchor, true);const afterDuplicate = await paintedViewport();@@ -364,29 +364,32 @@ describe("Live View canvas pinch zoom", function () {expect(existsSync(screenshot)).toBe(true);});- it("ignores an app-wide pinch while the canvas window is not focused", async () => {- // native-magnify is emitted app-wide from the gesture recognizer, so a- // canvas that does not have focus must not move.- const focused = (await browser.execute(() =>- document.hasFocus(),- )) as boolean;- if (focused) {- // If a runner ever does grant focus, assert the gesture is honoured- // rather than silently skipping the case.- const before = await paintedViewport();- await pinchBurst(0.02, 6);- const after = await paintedViewport();- expect(after.zoom).toBeGreaterThan(before.zoom);- expect(after.zoom).toBeLessThan(before.zoom * 1.5);- return;- }+ it("honours a window-scoped pinch", async () => {+ const before = await paintedViewport();+ await pinchBurst(0.02, 6);+ const after = await paintedViewport();+ expect(after.zoom).toBeGreaterThan(before.zoom);+ expect(after.zoom).toBeLessThan(before.zoom * 1.5);+ });+ it("keeps ordinary pane, Block, and double clicks at the same zoom", async () => {const before = await paintedViewport();- await pinchBurst(0.02, 12);+ await browser.execute(() => {+ const pane = document.querySelector<HTMLElement>(".react-flow__pane");+ const block = document.querySelector<HTMLElement>(+ "[data-testid='canvas-block-tracked-work']",+ );+ if (!pane || !block) throw new Error("canvas click targets not found");+ pane.dispatchEvent(new MouseEvent("click", { bubbles: true }));+ pane.dispatchEvent(new MouseEvent("dblclick", { bubbles: true }));+ block.dispatchEvent(new MouseEvent("click", { bubbles: true }));+ });+ await browser.pause(t(200));+const after = await paintedViewport();- expect(after.zoom).toBeCloseTo(before.zoom, 5);expect(after.x).toBeCloseTo(before.x, 5);expect(after.y).toBeCloseTo(before.y, 5);+ expect(after.zoom).toBeCloseTo(before.zoom, 5);});it("keeps canvas zoom bounded, anchored, and persisted", async () => {diff --git a/apps/screenpipe-app-tauri/src-tauri/src/window/gesture.rs b/apps/screenpipe-app-tauri/src-tauri/src/window/gesture.rsindex 1bc364213..4288d8563 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/window/gesture.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/window/gesture.rs@@ -15,8 +15,8 @@//// The fix: attach an NSMagnificationGestureRecognizer to the panel's content// view. Gesture recognizers fire at the view level, continuously, for every-// pinch gesture. The handler emits "native-magnify" Tauri events that the-// JS timeline code listens for.+// pinch gesture. The handler emits a window-scoped "native-magnify" Tauri+// event that the JS surface under the gesture listens for.// ---------------------------------------------------------------------------#[cfg(target_os = "macos")]@@ -303,12 +303,45 @@ pub(crate) async fn history_swipe_navigation_enabled(pub(crate) static MAGNIFY_APP_HANDLE: std::sync::OnceLock<tauri::AppHandle> =std::sync::OnceLock::new();+#[cfg(target_os = "macos")]+unsafe fn emit_native_magnify_from_recognizer(+ app: &tauri::AppHandle,+ recognizer: *mut objc::runtime::Object,+ magnification: f64,+) {+ use objc::{msg_send, sel, sel_impl};+ use tauri::{Emitter, Manager};+ use tauri_nspanel::cocoa::base::{id, nil};++ let source_view: id = msg_send![recognizer, view];+ let source_window: id = if source_view == nil {+ nil+ } else {+ msg_send![source_view, window]+ };++ if source_window != nil {+ for window in app.webview_windows().values() {+ if window+ .ns_window()+ .is_ok_and(|native_window| native_window as id == source_window)+ {+ let _ = window.emit("native-magnify", magnification);+ return;+ }+ }+ }++ // Preserve pinch support for an unexpected window shape. The canvas also+ // requires the pointer to be inside it, so the fallback remains bounded.+ let _ = app.emit("native-magnify", magnification);+}+/// Call once during app setup to store the AppHandle for the magnify handler.#[cfg(target_os = "macos")]pub fn init_magnify_handler(app: tauri::AppHandle) {use objc::declare::ClassDecl;use objc::runtime::{Class, Object, Sel};- use tauri::Emitter;let _ = MAGNIFY_APP_HANDLE.set(app);@@ -328,7 +361,7 @@ pub fn init_magnify_handler(app: tauri::AppHandle) {// Reset so next callback gives delta, not cumulativelet _: () = msg_send![recognizer, setMagnification: 0.0f64];if let Some(app) = MAGNIFY_APP_HANDLE.get() {- let _ = app.emit("native-magnify", magnification);+ emit_native_magnify_from_recognizer(app, recognizer, magnification);}});}));