YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
route Live View Pipe requests to the agent
Louis Beaumontb25a06bd6e73
diff --git a/apps/screenpipe-app-tauri/components/settings/__tests__/brain-overview.test.tsx b/apps/screenpipe-app-tauri/components/settings/__tests__/brain-overview.test.tsxindex 9d688456b..2c1e63790 100644--- a/apps/screenpipe-app-tauri/components/settings/__tests__/brain-overview.test.tsx+++ b/apps/screenpipe-app-tauri/components/settings/__tests__/brain-overview.test.tsx@@ -23,6 +23,7 @@ const mocks = vi.hoisted(() => ({loadBrainViewCanvas: vi.fn(),saveBrainViewCanvas: vi.fn(),generateLiveViewWithPi: vi.fn(),+ showChatWithPrefill: vi.fn(),createOnboardingLiveView: vi.fn(),localFetch: vi.fn(),toast: vi.fn(),@@ -139,6 +140,9 @@ vi.mock("@/components/rewind/ai-presets-selector", () => ({vi.mock("@/lib/live-views/generate-live-view-with-pi", () => ({generateLiveViewWithPi: mocks.generateLiveViewWithPi,}));+vi.mock("@/lib/chat-utils", () => ({+ showChatWithPrefill: mocks.showChatWithPrefill,+}));vi.mock("@/lib/live-views/onboarding-live-view", () => ({createOnboardingLiveView: mocks.createOnboardingLiveView,}));@@ -261,6 +265,7 @@ beforeEach(() => {data: [],});mocks.refetchPipes.mockResolvedValue(undefined);+ mocks.showChatWithPrefill.mockResolvedValue(undefined);mocks.deleteBrainView.mockResolvedValue({ status: "ok", data: null });mocks.loadBrainViewCanvas.mockResolvedValue({ status: "ok", data: null });mocks.saveBrainViewCanvas.mockImplementation(async (request) => ({@@ -295,6 +300,10 @@ describe("inferLiveViewGenerationIntent", () => {["make a sales dashboard", true, "replace-dashboard"],["rebuild this around projects", true, "replace-dashboard"],["remove the focus chart", true, "replace-dashboard"],+ ["create a Pipe for project switches", true, "pipe-agent"],+ ["edit the daily-summary pipe schedule", true, "pipe-agent"],+ ["fix my weekly piep", false, "pipe-agent"],+ ["show results from the daily-summary pipe", true, "replace-dashboard"],["add time by app", false, "new-dashboard"],] as const)("maps %s with current view=%s",@@ -1395,6 +1404,60 @@ describe("BrainOverview", () => {);});+ it("hands explicit Pipe authoring requests to the full agent with a bounded Live View reference", async () => {+ mocks.listBrainViews.mockResolvedValue({+ status: "ok",+ data: [populatedView],+ });+ render(<BrainOverview />);++ const request = "create a Pipe that tracks project switches";+ fireEvent.change(+ await screen.findByPlaceholderText(/Ask AI to change this Live View/),+ { target: { value: request } },+ );++ expect(screen.getByTestId("live-view-generation-intent").textContent).toBe(+ "will open the agent for “How I worked today”",+ );+ const button = screen.getByTestId("live-view-ai-generate");+ expect(button.textContent).toContain("open agent");+ fireEvent.click(button);++ await waitFor(() =>+ expect(mocks.showChatWithPrefill).toHaveBeenCalledWith(+ expect.objectContaining({+ context: "Live View “How I worked today” (revision 3)",+ displayLabel: request,+ autoSend: true,+ source: "live-view-pipe-agent",+ useHomeChat: true,+ prompt: expect.stringContaining(+ 'Current Live View reference: {"id":"my-overview","title":"How I worked today","revision":3}',+ ),+ }),+ ),+ );+ const agentPrompt = mocks.showChatWithPrefill.mock.calls[0][0].prompt;+ expect(agentPrompt).toContain("screenpipe-cli skill");+ expect(agentPrompt).toContain("screenpipe_live_view tool");+ expect(agentPrompt).toContain("Ask for explicit confirmation");+ expect(agentPrompt).toContain("Use ask_user when it is available");+ expect(mocks.generateLiveViewWithPi).not.toHaveBeenCalled();+ expect(mocks.capture).toHaveBeenCalledWith(+ "live_view_pipe_agent_handoff",+ expect.objectContaining({+ has_current_view: true,+ current_block_count: 1,+ prompt_length: request.length,+ }),+ );+ const analyticsProperties = mocks.capture.mock.calls.find(+ ([event]) => event === "live_view_pipe_agent_handoff",+ )?.[1];+ expect(analyticsProperties).not.toHaveProperty("prompt");+ });+it("locks dashboard navigation while AI generation is in progress", async () => {mocks.listBrainViews.mockResolvedValue({status: "ok",diff --git a/apps/screenpipe-app-tauri/components/settings/brain-overview.tsx b/apps/screenpipe-app-tauri/components/settings/brain-overview.tsxindex c451319a6..02b0e317a 100644--- a/apps/screenpipe-app-tauri/components/settings/brain-overview.tsx+++ b/apps/screenpipe-app-tauri/components/settings/brain-overview.tsx@@ -62,6 +62,7 @@ import { useSettings } from "@/lib/hooks/use-settings";import { useHealthCheck } from "@/lib/hooks/use-health-check";import { useToast } from "@/components/ui/use-toast";import { localFetch } from "@/lib/api";+import { showChatWithPrefill } from "@/lib/chat-utils";import {readActiveAiPresetId,resolveActiveAiPreset,@@ -1379,6 +1380,61 @@ export function BrainOverview({preset: AIPreset,intent: LiveViewGenerationIntent,) => {+ if (intent === "pipe-agent") {+ posthog.capture("live_view_pipe_agent_handoff", {+ analytics_schema_version: LIVE_VIEW_ANALYTICS_SCHEMA_VERSION,+ has_current_view: Boolean(view),+ current_block_count: view?.slots.length ?? 0,+ prompt_length: prompt.length,+ });++ const liveViewReference = view+ ? JSON.stringify({+ id: view.id,+ title: view.title,+ revision: view.revision,+ })+ : "none; the user has not created a Live View yet";+ const agentPrompt = `The user started this request from Brain > Live Views.++Current Live View reference: ${liveViewReference}++User request:+${prompt}++Use the screenpipe-cli skill for Pipe creation or editing and the screenpipe_live_view tool for Live View work.+- Read the screenpipe-cli skill before changing a Pipe.+- Load the current Live View lazily by id. Do not ask the app to inject its contents into chat context.+- It is okay to create a new disabled/manual Pipe draft and run it once for testing.+- Ask for explicit confirmation before overwriting or deleting an existing Pipe, enabling or scheduling automatic runs, installing anything, or replacing Live View Blocks.+- Use ask_user when it is available; otherwise ask one short question in chat and wait.+- Test a changed Pipe with pipe run before binding it to the Live View.+- Preserve unchanged Live View Blocks and bind only the relevant Blocks after the Pipe works.+- Do not publish anything to the Pipe Store.`;++ try {+ await showChatWithPrefill({+ context: view+ ? `Live View “${view.title}” (revision ${view.revision})`+ : "Create a Pipe for a new Live View",+ prompt: agentPrompt,+ displayLabel: prompt,+ autoSend: true,+ source: "live-view-pipe-agent",+ useHomeChat: true,+ });+ } catch (handoffError) {+ toast({+ title: "could not open the Pipe agent",+ description:+ handoffError instanceof Error+ ? handoffError.message+ : String(handoffError),+ variant: "destructive",+ });+ }+ return;+ }await generate(prompt, scope, preset, intent);};diff --git a/apps/screenpipe-app-tauri/components/settings/live-view-ai-composer.tsx b/apps/screenpipe-app-tauri/components/settings/live-view-ai-composer.tsxindex b5064c83a..9119f1c6d 100644--- a/apps/screenpipe-app-tauri/components/settings/live-view-ai-composer.tsx+++ b/apps/screenpipe-app-tauri/components/settings/live-view-ai-composer.tsx@@ -40,12 +40,22 @@ type LiveViewAiComposerProps = {) => void | Promise<void>;};-export type LiveViewGenerationIntent = "new-dashboard" | "replace-dashboard";+export type LiveViewGenerationIntent =+ "new-dashboard" | "replace-dashboard" | "pipe-agent";++const PIPE_NOUN = /\b(?:pipe|pipes|piep|pieps)\b/i;+const PIPE_AUTHORING_ACTION =+ /\b(?:create|make|build|add|new|edit|change|modify|update|fix|improve|optimi[sz]e|customi[sz]e|fork|remix)\b/i;++export function isPipeAuthoringRequest(prompt: string): boolean {+ return PIPE_NOUN.test(prompt) && PIPE_AUTHORING_ACTION.test(prompt);+}export function inferLiveViewGenerationIntent(- _prompt: string,+ prompt: string,hasCurrentView: boolean,): LiveViewGenerationIntent {+ if (isPipeAuthoringRequest(prompt)) return "pipe-agent";return hasCurrentView ? "replace-dashboard" : "new-dashboard";}@@ -86,9 +96,13 @@ export function LiveViewAiComposer({);const scope: LiveViewGenerationScope = "dashboard";const intentLabel =- intent === "replace-dashboard"- ? `will edit “${currentViewTitle}”`- : "will create a new dashboard";+ intent === "pipe-agent"+ ? currentViewTitle+ ? `will open the agent for “${currentViewTitle}”`+ : "will open the Pipe agent"+ : intent === "replace-dashboard"+ ? `will edit “${currentViewTitle}”`+ : "will create a new dashboard";const submit = () => {if (!canSubmit || !selectedPreset) return;@@ -169,7 +183,9 @@ export function LiveViewAiComposer({</>) : (<>- <span className="mr-1.5">generate</span>+ <span className="mr-1.5">+ {intent === "pipe-agent" ? "open agent" : "generate"}+ </span><ArrowUp className="h-3.5 w-3.5" /></>)}