YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
make Live View time and canvas gestures automatic
Louis Beaumontd5ad6ecbd09e
diff --git a/apps/screenpipe-app-tauri/components/settings/__tests__/brain-overview.test.tsx b/apps/screenpipe-app-tauri/components/settings/__tests__/brain-overview.test.tsxindex be1af1093..0f0982abe 100644--- a/apps/screenpipe-app-tauri/components/settings/__tests__/brain-overview.test.tsx+++ b/apps/screenpipe-app-tauri/components/settings/__tests__/brain-overview.test.tsx@@ -830,7 +830,7 @@ describe("BrainOverview", () => {expect(screen.queryByTestId("overview-time-range")).toBeNull();expect(screen.getByText(- "Pipes update these Blocks when you refresh or a connected Pipe runs.",+ /Pipes fill these Blocks for today\. Data changes when you refresh or a connected Pipe runs\./,),).toBeTruthy();@@ -1317,6 +1317,10 @@ describe("BrainOverview", () => {mocks.generateLiveViewWithPi.mockResolvedValue({title: "My working week",timeRange: "7d",+ periodPolicy: {+ type: "selectable.v1",+ values: ["today", "24h", "7d", "30d"],+ },note: "A time overview with automation opportunities.",blocks: [{@@ -1368,6 +1372,10 @@ describe("BrainOverview", () => {fireEvent.click(screen.getByTestId("overview-apply-ai"));await waitFor(() => expect(mocks.saveBrainView).toHaveBeenCalledTimes(2));+ expect(mocks.saveBrainView.mock.calls[1][0].periodPolicy).toEqual({+ type: "selectable.v1",+ values: ["today", "24h", "7d", "30d"],+ });expect(mocks.saveBrainView.mock.calls[1][0].slots).toEqual([expect.objectContaining({title: "Time by project",diff --git a/apps/screenpipe-app-tauri/components/settings/__tests__/live-view-canvas.test.tsx b/apps/screenpipe-app-tauri/components/settings/__tests__/live-view-canvas.test.tsxindex 63825855e..8c81522ad 100644--- a/apps/screenpipe-app-tauri/components/settings/__tests__/live-view-canvas.test.tsx+++ b/apps/screenpipe-app-tauri/components/settings/__tests__/live-view-canvas.test.tsx@@ -4,7 +4,13 @@import React, { useState } from "react";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";-import { fireEvent, render, screen, waitFor } from "@testing-library/react";+import {+ act,+ fireEvent,+ render,+ screen,+ waitFor,+} from "@testing-library/react";import { LiveViewCanvas } from "../live-view-canvas";import { createCanvasDocument } from "@/lib/live-views/canvas-layout";import type {@@ -12,6 +18,15 @@ import type {BrainViewDefinition,} from "@/lib/utils/tauri";+const eventMocks = vi.hoisted(() => ({+ listeners: new Map<string, (event: { payload: unknown }) => void>(),+ listen: vi.fn(),+}));++vi.mock("@tauri-apps/api/event", () => ({+ listen: eventMocks.listen,+}));+const view: BrainViewDefinition = {id: "canvas-component-view",title: "Canvas component",@@ -107,6 +122,17 @@ function CanvasHarness({}beforeEach(() => {+ eventMocks.listeners.clear();+ eventMocks.listen.mockClear();+ eventMocks.listen.mockImplementation(+ async (+ event: string,+ handler: (event: { payload: unknown }) => void,+ ) => {+ eventMocks.listeners.set(event, handler);+ return () => eventMocks.listeners.delete(event);+ },+ );vi.stubGlobal("PointerEvent", PointerEventMock);HTMLElement.prototype.setPointerCapture = vi.fn();HTMLElement.prototype.releasePointerCapture = vi.fn();@@ -268,4 +294,23 @@ describe("LiveViewCanvas", () => {}expect(screen.getByText("25%")).toBeTruthy();});++ it("zooms in and out around the trackpad pinch point", async () => {+ render(<CanvasHarness />);+ const surface = screen.getByTestId("live-view-canvas-surface");+ fireEvent.pointerEnter(surface, { clientX: 250, clientY: 200 });+ await waitFor(() =>+ expect(eventMocks.listeners.has("native-magnify")).toBe(true),+ );++ act(() => {+ eventMocks.listeners.get("native-magnify")?.({ payload: 0.1 });+ });+ expect(screen.getByText("165%")).toBeTruthy();++ act(() => {+ eventMocks.listeners.get("native-magnify")?.({ payload: -0.1 });+ });+ expect(screen.getByText("100%")).toBeTruthy();+ });});diff --git a/apps/screenpipe-app-tauri/components/settings/brain-overview.tsx b/apps/screenpipe-app-tauri/components/settings/brain-overview.tsxindex 5acf81125..3886d71eb 100644--- a/apps/screenpipe-app-tauri/components/settings/brain-overview.tsx+++ b/apps/screenpipe-app-tauri/components/settings/brain-overview.tsx@@ -1317,7 +1317,8 @@ export function BrainOverview({title: generated.title,revision: 0,timeRange: generated.timeRange,- periodPolicy: DEFAULT_LIVE_VIEW_PERIOD_POLICY,+ periodPolicy:+ generated.periodPolicy ?? DEFAULT_LIVE_VIEW_PERIOD_POLICY,slots: [],createdAt: now,updatedAt: now,@@ -2473,9 +2474,7 @@ export function BrainOverview({<p className="mt-2 text-xs text-muted-foreground">{onboardingColdStart? "This view will appear when Screenpipe has enough real activity for your outcome."- : view.periodPolicy.type === "fixed.v1"- ? "Pipes update these Blocks when you refresh or a connected Pipe runs."- : `Pipes fill these Blocks for ${selectedPeriod.label.toLowerCase()}. Data changes when you refresh or a connected Pipe runs.`}+ : `Pipes fill these Blocks for ${selectedPeriod.label.toLowerCase()}. Data changes when you refresh or a connected Pipe runs.`}{latestDataTimestamp !== null && (<span className="ml-1">Last data {new Date(latestDataTimestamp).toLocaleString()}.diff --git a/apps/screenpipe-app-tauri/components/settings/live-view-canvas.tsx b/apps/screenpipe-app-tauri/components/settings/live-view-canvas.tsxindex 58c54c10b..7e18a5576 100644--- a/apps/screenpipe-app-tauri/components/settings/live-view-canvas.tsx+++ b/apps/screenpipe-app-tauri/components/settings/live-view-canvas.tsx@@ -10,6 +10,7 @@ import React, {useRef,useState,} from "react";+import { listen } from "@tauri-apps/api/event";import {Background,BackgroundVariant,@@ -366,6 +367,9 @@ export function LiveViewCanvas({);const surfaceRef = useRef<HTMLDivElement | null>(null);const drawSessionRef = useRef<DrawSession | null>(null);+ const canvasActiveRef = useRef(false);+ const pointerAnchorRef = useRef<BrainViewCanvasPoint | null>(null);+ const nativeZoomEndRef = useRef<ReturnType<typeof setTimeout> | null>(null);const latestDocumentRef = useRef(document);const slotsById = useMemo(() => new Map(slots.map((slot) => [slot.id, slot])),@@ -815,6 +819,7 @@ export function LiveViewCanvas({const handleSurfacePointerMove = useCallback((event: React.PointerEvent<HTMLDivElement>) => {+ pointerAnchorRef.current = { x: event.clientX, y: event.clientY };const session = drawSessionRef.current;if (!session || session.pointerId !== event.pointerId) return;event.preventDefault();@@ -904,30 +909,66 @@ export function LiveViewCanvas({}, [applyDocument, selection, setCanvasSelection]);const zoomCanvas = useCallback(- (factor: number) => {+ (+ factor: number,+ anchor: BrainViewCanvasPoint | null = null,+ persist = true,+ ) => {const current = latestDocumentRef.current;const bounds = surfaceRef.current?.getBoundingClientRect();const zoom = clampCanvasZoom(current.viewport.zoom * factor);if (!bounds || zoom === current.viewport.zoom) return;- const centerX = bounds.width / 2;- const centerY = bounds.height / 2;- const worldX = (centerX - current.viewport.x) / current.viewport.zoom;- const worldY = (centerY - current.viewport.y) / current.viewport.zoom;+ const anchorX = anchor+ ? Math.min(bounds.width, Math.max(0, anchor.x - bounds.left))+ : bounds.width / 2;+ const anchorY = anchor+ ? Math.min(bounds.height, Math.max(0, anchor.y - bounds.top))+ : bounds.height / 2;+ const worldX = (anchorX - current.viewport.x) / current.viewport.zoom;+ const worldY = (anchorY - current.viewport.y) / current.viewport.zoom;applyDocument({...current,viewport: {- x: centerX - worldX * zoom,- y: centerY - worldY * zoom,+ x: anchorX - worldX * zoom,+ y: anchorY - worldY * zoom,zoom,},},- true,+ persist,);},[applyDocument],);+ useEffect(() => {+ const unlisten = listen<number>("native-magnify", (event) => {+ const magnification = event.payload;+ if (+ !canvasActiveRef.current ||+ typeof magnification !== "number" ||+ !Number.isFinite(magnification) ||+ magnification === 0+ ) {+ return;+ }+ zoomCanvas(+ Math.exp(magnification * 5),+ pointerAnchorRef.current,+ false,+ );+ if (nativeZoomEndRef.current) clearTimeout(nativeZoomEndRef.current);+ nativeZoomEndRef.current = setTimeout(() => {+ applyDocument(latestDocumentRef.current, true);+ nativeZoomEndRef.current = null;+ }, 160);+ });+ return () => {+ if (nativeZoomEndRef.current) clearTimeout(nativeZoomEndRef.current);+ void unlisten.then((dispose) => dispose());+ };+ }, [applyDocument, zoomCanvas]);+const fitCanvas = useCallback(() => {const current = latestDocumentRef.current;const surface = surfaceRef.current?.getBoundingClientRect();@@ -1074,6 +1115,17 @@ export function LiveViewCanvas({aria-label="Whiteboard canvas. Use the toolbar to select, pan, add notes, connect Blocks, or draw."tabIndex={0}className="live-view-process-canvas absolute inset-0 outline-none focus-visible:ring-1 focus-visible:ring-foreground focus-visible:ring-inset"+ onPointerEnter={(event) => {+ canvasActiveRef.current = true;+ pointerAnchorRef.current = {+ x: event.clientX,+ y: event.clientY,+ };+ }}+ onPointerLeave={() => {+ canvasActiveRef.current = false;+ pointerAnchorRef.current = null;+ }}onPointerDownCapture={handleSurfacePointerDown}onPointerMoveCapture={handleSurfacePointerMove}onPointerUpCapture={finishDrawSession}@@ -1264,7 +1316,7 @@ export function LiveViewCanvas({</div>)}<div className="pointer-events-none absolute bottom-3 right-3 z-20 border border-border bg-background/95 px-2 py-1 font-mono text-[9px] text-muted-foreground">- drag nodes · pan tool or middle-drag · ctrl/⌘ + wheel to zoom+ drag nodes · pan tool or middle-drag · pinch or ctrl/⌘ + wheel to zoom</div></section>);diff --git a/apps/screenpipe-app-tauri/e2e/specs/brain-overview.spec.ts b/apps/screenpipe-app-tauri/e2e/specs/brain-overview.spec.tsindex 52332e28d..c0c9d2f44 100644--- a/apps/screenpipe-app-tauri/e2e/specs/brain-overview.spec.ts+++ b/apps/screenpipe-app-tauri/e2e/specs/brain-overview.spec.ts@@ -608,6 +608,12 @@ Refresh the assigned Live View output targets from source-backed activity.expect(await $("[data-testid='overview-time-range']").isExisting()).toBe(false,);+ const fixedDashboardText = (await browser.execute(+ () => document.body?.innerText || "",+ )) as string;+ expect(fixedDashboardText).toContain(+ "Pipes fill these Blocks for today. Data changes when you refresh or a connected Pipe runs.",+ );const fixedScreenshot = await saveScreenshot("brain-overview-fixed-range-hidden",);diff --git a/apps/screenpipe-app-tauri/lib/live-views/__tests__/generate-live-view-with-pi.test.ts b/apps/screenpipe-app-tauri/lib/live-views/__tests__/generate-live-view-with-pi.test.tsindex 6c4d023d0..134b155cf 100644--- a/apps/screenpipe-app-tauri/lib/live-views/__tests__/generate-live-view-with-pi.test.ts+++ b/apps/screenpipe-app-tauri/lib/live-views/__tests__/generate-live-view-with-pi.test.ts@@ -14,6 +14,7 @@ describe("parseGeneratedLiveView", () => {JSON.stringify({title: "How I worked",timeRange: "7d",+ timeRangeBehavior: "selectable",note: "A useful overview.",blocks: [{@@ -53,6 +54,10 @@ describe("parseGeneratedLiveView", () => {expect(result).toEqual({title: "How I worked",timeRange: "7d",+ periodPolicy: {+ type: "selectable.v1",+ values: ["today", "24h", "7d", "30d"],+ },note: "A useful overview.",blocks: [{@@ -87,6 +92,29 @@ describe("parseGeneratedLiveView", () => {});});+ it("automatically fixes the period only when the generated view requires it", () => {+ const result = parseGeneratedLiveView(+ JSON.stringify({+ title: "Daily memory",+ timeRange: "today",+ timeRangeBehavior: "fixed",+ blocks: [+ {+ title: "Daily brief",+ component: "markdown.v1",+ },+ ],+ }),+ [],+ "dashboard",+ );++ expect(result.periodPolicy).toEqual({+ type: "fixed.v1",+ value: "today",+ });+ });+it("strips JSON fences and refuses invented Pipe bindings", () => {const result = parseGeneratedLiveView('```json\n{"title":"Today","blocks":[{"title":"Tasks","type":"list","width":12,"pipe":"invented-pipe"}]}\n```',diff --git a/apps/screenpipe-app-tauri/lib/live-views/generate-live-view-with-pi.ts b/apps/screenpipe-app-tauri/lib/live-views/generate-live-view-with-pi.tsindex b57eaf446..630a98939 100644--- a/apps/screenpipe-app-tauri/lib/live-views/generate-live-view-with-pi.ts+++ b/apps/screenpipe-app-tauri/lib/live-views/generate-live-view-with-pi.ts@@ -7,6 +7,7 @@ import {commands,type AIPreset,type BrainViewComponent,+ type BrainViewPeriodPolicy,type BrainViewTimeRange,type PiProviderConfig,} from "@/lib/utils/tauri";@@ -60,6 +61,7 @@ export type GeneratedLiveViewBlock = {export type GeneratedLiveView = {title: string;timeRange: BrainViewTimeRange;+ periodPolicy?: BrainViewPeriodPolicy;blocks: GeneratedLiveViewBlock[];note: string;};@@ -132,6 +134,22 @@ function timeRangeValue(value: unknown): BrainViewTimeRange {return "today";}+function periodPolicyValue(+ value: unknown,+ timeRange: BrainViewTimeRange,+): BrainViewPeriodPolicy {+ if (+ typeof value === "string" &&+ value.trim().toLowerCase() === "fixed"+ ) {+ return { type: "fixed.v1", value: timeRange };+ }+ return {+ type: "selectable.v1",+ values: ["today", "24h", "7d", "30d"],+ };+}+export function parseGeneratedLiveView(raw: string,allowedPipeNames: string[],@@ -193,9 +211,14 @@ export function parseGeneratedLiveView(? `Created ${blocks[0].title}.`: `Created ${blocks.length} sections.`;+ const timeRange = timeRangeValue(parsed.timeRange ?? parsed.time_range);return {title,- timeRange: timeRangeValue(parsed.timeRange ?? parsed.time_range),+ timeRange,+ periodPolicy: periodPolicyValue(+ parsed.timeRangeBehavior ?? parsed.time_range_behavior,+ timeRange,+ ),blocks,note,};@@ -219,11 +242,11 @@ Allowed widths are 3, 6, or 12. Prefer 6 for most sections, 12 for timelines orOnly use a pipeName from the available pipes supplied by the user. Use null when none fits. Do not invent pipes.Every section must include an intent: one precise, self-contained sentence describing what the Pipe should calculate, classify, or summarize. The intent is data logic, not display copy. Define percentages and scores explicitly, name the selected-period denominator, require source evidence, and say how to handle unclassified or missing evidence. Never use a vague intent such as "show this metric".For a whole dashboard, create 4 to 7 distinct sections. Prefer a useful mix with at least one metric, one bar chart, and one list or timeline when the request supports them. Never return placeholder titles such as "test", duplicate sections, or multiple metrics that show the same number.-Choose one timeRange for the whole dashboard: "today", "24h", "7d", or "30d". Infer it from the request. Use "today" when the request does not specify a period. Prefer line-chart.v1 over bar-chart.v1 when the user asks how something changed over time.+Choose one timeRange for the whole dashboard: "today", "24h", "7d", or "30d". Infer it from the request. Use "today" when the request does not specify a period. Also choose timeRangeBehavior. Use "fixed" only when the dashboard's identity is inherently tied to that exact period, such as a daily memory or a standup for the last 24 hours. Use "selectable" when the same analysis remains useful across different periods. If unsure, use "selectable". This choice is automatic and is never presented to the user as a setup decision. Prefer line-chart.v1 over bar-chart.v1 when the user asks how something changed over time.For one section, return exactly one focused section.Required JSON shape:-{"title":"View title","timeRange":"today","blocks":[{"title":"Section title","intent":"Precise source-backed question or calculation for this section and selected period.","component":"metric.v1","width":6,"pipeName":"exact-installed-pipe-name-or-null"}],"note":"One short sentence explaining what you created"}`;+{"title":"View title","timeRange":"today","timeRangeBehavior":"selectable","blocks":[{"title":"Section title","intent":"Precise source-backed question or calculation for this section and selected period.","component":"metric.v1","width":6,"pipeName":"exact-installed-pipe-name-or-null"}],"note":"One short sentence explaining what you created"}`;}function generationPrompt(options: GenerateLiveViewOptions): string {diff --git a/apps/screenpipe-app-tauri/lib/live-views/onboarding-live-view.ts b/apps/screenpipe-app-tauri/lib/live-views/onboarding-live-view.tsindex 7f5eb9d39..d07e94d32 100644--- a/apps/screenpipe-app-tauri/lib/live-views/onboarding-live-view.ts+++ b/apps/screenpipe-app-tauri/lib/live-views/onboarding-live-view.ts@@ -598,7 +598,8 @@ async function saveFirstDashboard(title,expectedRevision: existing?.revision ?? null,timeRange: generated.timeRange,- periodPolicy: DEFAULT_LIVE_VIEW_PERIOD_POLICY,+ periodPolicy:+ generated.periodPolicy ?? DEFAULT_LIVE_VIEW_PERIOD_POLICY,slots: slotInputs(generated, readyPipeNames),});if (saved.status === "error") {