YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(live-views): restore trackpad pinch zoom (#5901)
Ezra Ellette502dea1941e2
diff --git a/apps/screenpipe-app-tauri/components/settings/__tests__/live-view-canvas.test.tsx b/apps/screenpipe-app-tauri/components/settings/__tests__/live-view-canvas.test.tsxindex d34f44916..5ad95a267 100644--- a/apps/screenpipe-app-tauri/components/settings/__tests__/live-view-canvas.test.tsx+++ b/apps/screenpipe-app-tauri/components/settings/__tests__/live-view-canvas.test.tsx@@ -16,6 +16,15 @@ import type {BrainViewDefinition,} from "@/lib/utils/tauri";+const eventMocks = vi.hoisted(() => ({+ listeners: new Map<string, (event: { payload: unknown }) => void>(),+ listen: vi.fn(),+}));++vi.mock("@tauri-apps/api/event", () => ({+ listen: eventMocks.listen,+}));+const view: BrainViewDefinition = {id: "canvas-component-view",title: "Canvas component",@@ -89,9 +98,11 @@ class PointerEventMock extends MouseEvent {function CanvasHarness({initialDocument = createCanvasDocument(view),onPersist = vi.fn(),+ focusSlotId = null,}: {initialDocument?: BrainViewCanvasDocument;onPersist?: (document: BrainViewCanvasDocument) => void;+ focusSlotId?: string | null;}) {const [document, setDocument] = useState(initialDocument);return (@@ -101,6 +112,7 @@ function CanvasHarness({timeRange="today"refreshingSlotIds={new Set()}aiEditingSlotId={null}+ focusSlotId={focusSlotId}onChange={(next, options) => {setDocument(next);if (options.persist) onPersist(next);@@ -118,6 +130,17 @@ function openCanvasTools() {}beforeEach(() => {+ eventMocks.listeners.clear();+ eventMocks.listen.mockClear();+ eventMocks.listen.mockImplementation(+ async (+ event: string,+ handler: (event: { payload: unknown }) => void,+ ) => {+ eventMocks.listeners.set(event, handler);+ return () => eventMocks.listeners.delete(event);+ },+ );vi.stubGlobal("PointerEvent", PointerEventMock);HTMLElement.prototype.setPointerCapture = vi.fn();HTMLElement.prototype.releasePointerCapture = vi.fn();@@ -594,7 +617,7 @@ describe("LiveViewCanvas", () => {expect(screen.getByText("25%")).toBeTruthy();});- it("keeps wheel zoom behind the cmd or ctrl modifier", () => {+ it("keeps plain wheel as pan and recognizes browser trackpad pinch", () => {render(<CanvasHarness />);const surface = screen.getByTestId("live-view-canvas-surface");const pane = surface.querySelector<HTMLElement>(".react-flow__pane");@@ -604,13 +627,116 @@ describe("LiveViewCanvas", () => {fireEvent.wheel(pane!, { deltaY: -100, clientX: 500, clientY: 350 });expect(screen.getByText("100%")).toBeTruthy();- fireEvent.keyDown(document, {- key: "Meta",- code: "MetaLeft",- metaKey: true,+ fireEvent.wheel(pane!, {+ ctrlKey: true,+ deltaY: -100,+ clientX: 500,+ clientY: 350,});- fireEvent.wheel(pane!, { deltaY: -100, clientX: 500, clientY: 350 });expect(screen.queryByText("100%")).toBeNull();- fireEvent.keyUp(document, { key: "Meta", code: "MetaLeft" });+ });++ it("zooms native trackpad pinches around the pointer and persists once", async () => {+ const initialDocument = createCanvasDocument(view);+ const onPersist = vi.fn();+ vi.spyOn(document, "hasFocus").mockReturnValue(true);+ render(+ <CanvasHarness+ initialDocument={initialDocument}+ onPersist={onPersist}+ />,+ );+ const surface = screen.getByTestId("live-view-canvas-surface");+ fireEvent.pointerEnter(surface, { clientX: 250, clientY: 200 });+ openCanvasTools();+ await waitFor(() =>+ expect(eventMocks.listeners.has("native-magnify")).toBe(true),+ );++ act(() => {+ eventMocks.listeners.get("native-magnify")?.({ payload: 0.1 });+ });+ expect(screen.getByText("165%")).toBeTruthy();+ act(() => {+ eventMocks.listeners.get("native-magnify")?.({ payload: -0.1 });+ });+ expect(screen.getByText("100%")).toBeTruthy();++ await waitFor(() => expect(onPersist).toHaveBeenCalledTimes(1));+ const persisted = onPersist.mock.calls[0][0] as BrainViewCanvasDocument;+ expect(persisted.viewport.zoom).toBeCloseTo(+ initialDocument.viewport.zoom,+ );+ const worldBefore = {+ x:+ (250 - initialDocument.viewport.x) /+ initialDocument.viewport.zoom,+ y:+ (200 - initialDocument.viewport.y) /+ initialDocument.viewport.zoom,+ };+ expect((250 - persisted.viewport.x) / persisted.viewport.zoom).toBeCloseTo(+ worldBefore.x,+ );+ expect((200 - persisted.viewport.y) / persisted.viewport.zoom).toBeCloseTo(+ worldBefore.y,+ );+ });++ it("lets a native pinch take over an in-flight AI focus animation", async () => {+ vi.stubGlobal(+ "matchMedia",+ vi.fn(() => ({ matches: false })),+ );+ vi.spyOn(document, "hasFocus").mockReturnValue(true);+ // Hold the focus animation in flight: rAF hands back an id but never runs+ // its callback, so only a real pinch can end the animation.+ const cancelAnimationFrame = vi.fn();+ vi.stubGlobal(+ "requestAnimationFrame",+ vi.fn(() => 77),+ );+ vi.stubGlobal("cancelAnimationFrame", cancelAnimationFrame);++ const result = render(<CanvasHarness />);+ const surface = screen.getByTestId("live-view-canvas-surface");+ fireEvent.pointerEnter(surface, { clientX: 250, clientY: 200 });+ openCanvasTools();+ await waitFor(() =>+ expect(eventMocks.listeners.has("native-magnify")).toBe(true),+ );++ result.rerender(<CanvasHarness focusSlotId="focus-time" />);+ expect(cancelAnimationFrame).not.toHaveBeenCalledWith(77);++ act(() => {+ eventMocks.listeners.get("native-magnify")?.({ payload: 0.1 });+ });++ expect(cancelAnimationFrame).toHaveBeenCalledWith(77);+ expect(screen.getByText("165%")).toBeTruthy();+ });++ it("ignores app-wide native pinches outside the focused canvas", async () => {+ vi.spyOn(document, "hasFocus").mockReturnValue(false);+ render(<CanvasHarness />);+ const surface = screen.getByTestId("live-view-canvas-surface");+ fireEvent.pointerEnter(surface, { clientX: 250, clientY: 200 });+ openCanvasTools();+ await waitFor(() =>+ expect(eventMocks.listeners.has("native-magnify")).toBe(true),+ );++ act(() => {+ eventMocks.listeners.get("native-magnify")?.({ payload: 0.1 });+ });+ expect(screen.getByText("100%")).toBeTruthy();++ vi.mocked(document.hasFocus).mockReturnValue(true);+ fireEvent.pointerLeave(surface);+ act(() => {+ eventMocks.listeners.get("native-magnify")?.({ payload: 0.1 });+ });+ expect(screen.getByText("100%")).toBeTruthy();});});diff --git a/apps/screenpipe-app-tauri/components/settings/live-view-canvas.tsx b/apps/screenpipe-app-tauri/components/settings/live-view-canvas.tsxindex 4692b5a5c..0a75356fe 100644--- a/apps/screenpipe-app-tauri/components/settings/live-view-canvas.tsx+++ b/apps/screenpipe-app-tauri/components/settings/live-view-canvas.tsx@@ -51,6 +51,7 @@ import {type LiveViewListItem,} from "@/components/settings/live-view-card";import { Button } from "@/components/ui/button";+import { useTauriEvent } from "@/lib/hooks/use-tauri-event";import {CANVAS_GRID,canvasBlockNodeId,@@ -477,6 +478,9 @@ export function LiveViewCanvas({);const surfaceRef = useRef<HTMLDivElement | null>(null);const drawSessionRef = useRef<DrawSession | null>(null);+ const canvasActiveRef = useRef(false);+ const pointerAnchorRef = useRef<BrainViewCanvasPoint | null>(null);+ const nativeZoomEndRef = useRef<ReturnType<typeof setTimeout> | null>(null);const focusAnimationFrameRef = useRef<number | null>(null);const focusAnimationTimeoutRef = useRef<number | null>(null);const [focusViewport, setFocusViewport] = useState<Viewport | null>(null);@@ -1082,6 +1086,7 @@ export function LiveViewCanvas({const handleSurfacePointerMove = useCallback((event: React.PointerEvent<HTMLDivElement>) => {+ pointerAnchorRef.current = { x: event.clientX, y: event.clientY };const session = drawSessionRef.current;if (!session || session.pointerId !== event.pointerId) return;event.preventDefault();@@ -1171,30 +1176,70 @@ export function LiveViewCanvas({}, [applyDocument, selection, setCanvasSelection]);const zoomCanvas = useCallback(- (factor: number) => {+ (+ factor: number,+ anchor: BrainViewCanvasPoint | null = null,+ persist = true,+ ) => {const current = latestDocumentRef.current;const bounds = surfaceRef.current?.getBoundingClientRect();const zoom = clampCanvasZoom(current.viewport.zoom * factor);if (!bounds || zoom === current.viewport.zoom) return;- const centerX = bounds.width / 2;- const centerY = bounds.height / 2;- const worldX = (centerX - current.viewport.x) / current.viewport.zoom;- const worldY = (centerY - current.viewport.y) / current.viewport.zoom;+ const anchorX = anchor+ ? Math.min(bounds.width, Math.max(0, anchor.x - bounds.left))+ : bounds.width / 2;+ const anchorY = anchor+ ? Math.min(bounds.height, Math.max(0, anchor.y - bounds.top))+ : bounds.height / 2;+ const worldX = (anchorX - current.viewport.x) / current.viewport.zoom;+ const worldY = (anchorY - current.viewport.y) / current.viewport.zoom;applyDocument({...current,viewport: {- x: centerX - worldX * zoom,- y: centerY - worldY * zoom,+ x: anchorX - worldX * zoom,+ y: anchorY - worldY * zoom,zoom,},},- true,+ persist,);},[applyDocument],);+ useTauriEvent<number>("native-magnify", (event) => {+ const magnification = event.payload;+ if (+ !canvasActiveRef.current ||+ !window.document.hasFocus() ||+ typeof magnification !== "number" ||+ !Number.isFinite(magnification) ||+ magnification === 0+ ) {+ return;+ }+ // A real pinch takes over from an in-flight AI focus animation, the same+ // way a trusted pan or wheel zoom does.+ cancelFocusAnimation();+ zoomCanvas(+ Math.exp(magnification * 5),+ pointerAnchorRef.current,+ false,+ );+ if (nativeZoomEndRef.current) clearTimeout(nativeZoomEndRef.current);+ nativeZoomEndRef.current = setTimeout(() => {+ applyDocument(latestDocumentRef.current, true);+ nativeZoomEndRef.current = null;+ }, 160);+ });++ useEffect(() => {+ return () => {+ if (nativeZoomEndRef.current) clearTimeout(nativeZoomEndRef.current);+ };+ }, []);+const fitCanvas = useCallback(() => {const current = latestDocumentRef.current;const surface = surfaceRef.current?.getBoundingClientRect();@@ -1334,6 +1379,8 @@ export function LiveViewCanvas({const handleMoveEnd = useCallback((event: MouseEvent | TouchEvent | null, viewport: Viewport) => {if (!isTrustedCanvasMove(event)) return;+ // Native pinch persists once when its gesture burst settles.+ if (nativeZoomEndRef.current) return;const current = latestDocumentRef.current;applyDocument({ ...current, viewport }, true);},@@ -1358,6 +1405,17 @@ export function LiveViewCanvas({aria-label="Whiteboard canvas. Use the toolbar to select, pan, add notes, connect Blocks, or draw."tabIndex={0}className="live-view-process-canvas absolute inset-0 outline-none focus-visible:ring-1 focus-visible:ring-foreground focus-visible:ring-inset"+ onPointerEnter={(event) => {+ canvasActiveRef.current = true;+ pointerAnchorRef.current = {+ x: event.clientX,+ y: event.clientY,+ };+ }}+ onPointerLeave={() => {+ canvasActiveRef.current = false;+ pointerAnchorRef.current = null;+ }}onPointerDownCapture={handleSurfacePointerDown}onPointerMoveCapture={handleSurfacePointerMove}onPointerUpCapture={finishDrawSession}@@ -1627,7 +1685,8 @@ export function LiveViewCanvas({data-testid="canvas-interaction-hint"className="pointer-events-none absolute right-3 top-3 z-20 max-w-[calc(100%-11rem)] border border-border bg-background/95 px-2 py-1 text-right font-mono text-[9px] leading-tight text-muted-foreground">- drag nodes · pan tool or middle-drag · ctrl/⌘ + wheel to zoom+ drag nodes · pan tool or middle-drag · pinch or ctrl/⌘ + wheel to+ zoom</div>)}</section>