YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix: infer Live View generation intent
Louis Beaumont73e5264643ea
diff --git a/apps/screenpipe-app-tauri/components/settings/__tests__/brain-overview.test.tsx b/apps/screenpipe-app-tauri/components/settings/__tests__/brain-overview.test.tsxindex 2af017fbd..08156bc30 100644--- a/apps/screenpipe-app-tauri/components/settings/__tests__/brain-overview.test.tsx+++ b/apps/screenpipe-app-tauri/components/settings/__tests__/brain-overview.test.tsx@@ -84,6 +84,7 @@ vi.mock("@/lib/live-views/generate-live-view-with-pi", () => ({generateLiveViewWithPi: mocks.generateLiveViewWithPi,}));import { BrainOverview, type ViewDefinition } from "../brain-overview";+import { inferLiveViewGenerationIntent } from "../live-view-ai-composer";const populatedView: ViewDefinition = {id: "my-overview",@@ -171,6 +172,27 @@ beforeEach(() => {});});+describe("inferLiveViewGenerationIntent", () => {+ it.each([+ ["track my time", true, "new-dashboard"],+ ["add time by app", true, "add-section"],+ ["also show my meetings", true, "add-section"],+ ["show me one chart for project time", true, "add-section"],+ ["add a sales dashboard", true, "new-dashboard"],+ ["make a sales dashboard", true, "new-dashboard"],+ ["rebuild this around projects", true, "replace-dashboard"],+ ["remove the focus chart", true, "replace-dashboard"],+ ["add time by app", false, "new-dashboard"],+ ] as const)(+ "maps %s with current view=%s",+ (prompt, hasCurrentView, expected) => {+ expect(inferLiveViewGenerationIntent(prompt, hasCurrentView)).toBe(+ expected,+ );+ },+ );+});+describe("BrainOverview", () => {it("renders validated pipe data with its artifact provenance", async () => {mocks.listBrainViews.mockResolvedValue({@@ -813,17 +835,15 @@ describe("BrainOverview", () => {render(<BrainOverview />);const prompt = await screen.findByPlaceholderText(- /Ask AI to change this view/,+ /Ask AI to create a dashboard/,);- expect(- (screen.getByLabelText("generation scope") as HTMLSelectElement).value,- ).toBe("dashboard");- fireEvent.change(screen.getByLabelText("generation scope"), {- target: { value: "block" },- });+ expect(screen.queryByLabelText("generation scope")).toBeNull();fireEvent.change(prompt, {target: { value: "add work I could automate" },});+ expect(screen.getByTestId("live-view-generation-intent").textContent).toBe(+ "will add one section to “How I worked today”",+ );const generateButton = screen.getByTestId("live-view-ai-generate");await waitFor(() => expect(generateButton).not.toBeDisabled());fireEvent.click(generateButton);@@ -841,9 +861,15 @@ describe("BrainOverview", () => {});it("creates whole-dashboard AI drafts as a new named dashboard by default", async () => {+ const existingProjectPulse: ViewDefinition = {+ ...populatedView,+ id: "project-pulse",+ title: "Project pulse",+ revision: 1,+ };mocks.listBrainViews.mockResolvedValue({status: "ok",- data: [populatedView],+ data: [populatedView, existingProjectPulse],});mocks.generateLiveViewWithPi.mockResolvedValue({title: "Project pulse",@@ -875,13 +901,20 @@ describe("BrainOverview", () => {render(<BrainOverview />);fireEvent.change(- await screen.findByPlaceholderText(/Ask AI to change this view/),+ await screen.findByPlaceholderText(/Ask AI to create a dashboard/),{ target: { value: "build a separate project dashboard" } },);const generateButton = screen.getByTestId("live-view-ai-generate");await waitFor(() => expect(generateButton).not.toBeDisabled());fireEvent.click(generateButton);+ expect(mocks.generateLiveViewWithPi).toHaveBeenCalledWith(+ expect.objectContaining({+ scope: "dashboard",+ currentView: null,+ }),+ );+expect((await screen.findByTestId("overview-destination-new")).getAttribute("aria-pressed",@@ -892,8 +925,8 @@ describe("BrainOverview", () => {await waitFor(() => expect(mocks.saveBrainView).toHaveBeenCalledTimes(1));expect(mocks.saveBrainView.mock.calls[0][0]).toEqual(expect.objectContaining({- id: "project-pulse",- title: "Project pulse",+ id: "project-pulse-2",+ title: "Project pulse 2",expectedRevision: null,}),);@@ -938,9 +971,12 @@ describe("BrainOverview", () => {render(<BrainOverview />);fireEvent.change(- await screen.findByPlaceholderText(/Ask AI to change this view/),+ await screen.findByPlaceholderText(/Ask AI to create a dashboard/),{ target: { value: "rebuild this around projects" } },);+ expect(screen.getByTestId("live-view-generation-intent").textContent).toBe(+ "will preview changes to “How I worked today”",+ );const generateButton = screen.getByTestId("live-view-ai-generate");await waitFor(() => expect(generateButton).not.toBeDisabled());fireEvent.click(generateButton);@@ -950,10 +986,9 @@ describe("BrainOverview", () => {).toBeTruthy();expect(screen- .getByTestId("overview-destination-new")+ .getByTestId("overview-destination-replace").getAttribute("aria-pressed"),).toBe("true");- fireEvent.click(screen.getByTestId("overview-destination-replace"));expect(screen.getByTestId("overview-replacement-warning").textContent,).toContain("previous layout remains available through Undo");@@ -968,6 +1003,54 @@ describe("BrainOverview", () => {expect(mocks.saveBrainView).not.toHaveBeenCalled();});+ it("requires replace or delete when the dashboard limit is reached", async () => {+ const dashboardViews = Array.from({ length: 12 }, (_, index) => ({+ ...populatedView,+ id: `dashboard-${index + 1}`,+ title: `Dashboard ${index + 1}`,+ revision: 1,+ }));+ mocks.listBrainViews.mockResolvedValue({+ status: "ok",+ data: dashboardViews,+ });+ mocks.generateLiveViewWithPi.mockResolvedValue({+ title: "One more dashboard",+ timeRange: "today",+ note: "A new dashboard request.",+ blocks: [+ {+ title: "Active time",+ component: "metric.v1",+ width: 6,+ pipeName: "daily-summary",+ },+ ],+ });+ render(<BrainOverview />);++ fireEvent.change(+ await screen.findByPlaceholderText(/Ask AI to create a dashboard/),+ { target: { value: "track my time in a new way" } },+ );+ const generateButton = screen.getByTestId("live-view-ai-generate");+ await waitFor(() => expect(generateButton).not.toBeDisabled());+ fireEvent.click(generateButton);++ expect(+ (await screen.findByTestId("overview-destination-replace")).getAttribute(+ "aria-pressed",+ ),+ ).toBe("true");+ expect(screen.getByTestId("overview-destination-new")).toBeDisabled();+ expect(+ screen.getByTestId("overview-dashboard-limit-warning").textContent,+ ).toContain("Replace the current dashboard, or delete one");+ expect(screen.getByTestId("overview-apply-ai").textContent).toContain(+ "replace current dashboard",+ );+ });+it("offers per-card feedback and regenerates only that card", async () => {mocks.localFetch.mockImplementation(async (path: string) => ({ok: true,diff --git a/apps/screenpipe-app-tauri/components/settings/brain-overview.tsx b/apps/screenpipe-app-tauri/components/settings/brain-overview.tsxindex 70fbe7e9c..566a1adad 100644--- a/apps/screenpipe-app-tauri/components/settings/brain-overview.tsx+++ b/apps/screenpipe-app-tauri/components/settings/brain-overview.tsx@@ -15,7 +15,10 @@ import {X,} from "lucide-react";import { Button } from "@/components/ui/button";-import { LiveViewAiComposer } from "@/components/settings/live-view-ai-composer";+import {+ LiveViewAiComposer,+ type LiveViewGenerationIntent,+} from "@/components/settings/live-view-ai-composer";import { LiveViewCard as OverviewCard } from "@/components/settings/live-view-card";import { LiveViewDashboardSwitcher } from "@/components/settings/live-view-dashboard-switcher";import { LiveViewLayoutEditor } from "@/components/settings/live-view-layout-editor";@@ -599,7 +602,9 @@ export function BrainOverview() {updatedAt: now,});setPreviewSource({ kind: "template", kit });- setPreviewDestination("new");+ setPreviewDestination(+ view && views.length >= MAX_DASHBOARDS ? "replace" : "new",+ );setAiNote(kit.description);setEditing(false);setTemplateGalleryOpen(false);@@ -610,6 +615,7 @@ export function BrainOverview() {prompt: string,scope: LiveViewGenerationScope,preset: AIPreset,+ intent: LiveViewGenerationIntent,) => {setGenerating(true);try {@@ -624,18 +630,19 @@ export function BrainOverview() {pipe.prompt_body?.trim().slice(0, 500) ||`${pipe.config.name} Screenpipe Pipe`,})),- currentView: view- ? {- title: view.title,- timeRange: view.timeRange,- blocks: normalizedSlots(view.slots).map((slot) => ({- title: slot.title,- component: slot.component,- width: slot.width === 3 || slot.width === 12 ? slot.width : 6,- pipeName: slot.binding?.pipeName ?? null,- })),- }- : null,+ currentView:+ view && intent !== "new-dashboard"+ ? {+ title: view.title,+ timeRange: view.timeRange,+ blocks: normalizedSlots(view.slots).map((slot) => ({+ title: slot.title,+ component: slot.component,+ width: slot.width === 3 || slot.width === 12 ? slot.width : 6,+ pipeName: slot.binding?.pipeName ?? null,+ })),+ }+ : null,});const now = new Date().toISOString();const base: ViewDefinition = view@@ -666,7 +673,15 @@ export function BrainOverview() {});setAiNote(generated.note);setPreviewSource({ kind: "ai", scope });- setPreviewDestination(scope === "dashboard" ? "new" : "replace");+ setPreviewDestination(+ scope === "block"+ ? "replace"+ : view && views.length >= MAX_DASHBOARDS+ ? "replace"+ : intent === "replace-dashboard"+ ? "replace"+ : "new",+ );setEditing(false);setAiPreview(true);} catch (generateError) {@@ -878,12 +893,15 @@ export function BrainOverview() {}const previousView =!creatingNew && view ? copyViewDefinition(view) : null;+ const targetTitle = creatingNew+ ? uniqueDashboardTitle(draft.title.trim(), views)+ : draft.title.trim();const targetId = creatingNew- ? uniqueDashboardId(draft.title, views)+ ? uniqueDashboardId(targetTitle, views): draft.id;const result = await commands.saveBrainView({id: targetId,- title: draft.title.trim(),+ title: targetTitle,expectedRevision: creatingNew ? null : draft.revision,timeRange: draft.timeRange,slots: normalizedSlots(draft.slots).map((slot) => ({@@ -953,8 +971,11 @@ export function BrainOverview() {`You can keep up to ${MAX_DASHBOARDS} dashboards. Delete one before creating another.`,);}+ const requestedTitle = creatingNew+ ? uniqueDashboardTitle(draft?.title.trim() || kit.title, views)+ : draft?.title.trim() || kit.title;const targetViewId = creatingNew- ? uniqueDashboardId(draft?.title || kit.title, views)+ ? uniqueDashboardId(requestedTitle, views): view!.id;const result = await commands.installBrainViewTemplateKit({kitId: kit.id,@@ -963,7 +984,6 @@ export function BrainOverview() {});if (result.status === "error") throw new Error(result.error);let installedView = result.data;- const requestedTitle = draft?.title.trim();if (requestedTitle && requestedTitle !== installedView.title) {const renameResult = await commands.saveBrainView({id: installedView.id,@@ -1017,9 +1037,13 @@ export function BrainOverview() {if (!view || !title.trim() || title.trim() === view.title) return;setSaving(true);try {+ const uniqueTitle = uniqueDashboardTitle(+ title.trim(),+ views.filter((candidate) => candidate.id !== view.id),+ );const result = await commands.saveBrainView({id: view.id,- title: title.trim(),+ title: uniqueTitle,expectedRevision: view.revision,timeRange: view.timeRange,slots: normalizedSlots(view.slots).map((slot) => ({@@ -1268,6 +1292,7 @@ export function BrainOverview() {(previewSource?.kind === "ai" && previewSource.scope === "dashboard"),);const canChooseDestination = Boolean(view && wholeDashboardPreview);+ const dashboardLimitReached = views.length >= MAX_DASHBOARDS;const replacingDashboard = Boolean(canChooseDestination && previewDestination === "replace",);@@ -1344,7 +1369,13 @@ export function BrainOverview() {size="sm"variant={replacingDashboard ? "destructive" : "default"}className="rounded-none"- disabled={saving || !draft.title.trim()}+ disabled={+ saving ||+ !draft.title.trim() ||+ (wholeDashboardPreview &&+ destination === "new" &&+ dashboardLimitReached)+ }onClick={requestApply}>{saving && (@@ -1399,18 +1430,21 @@ export function BrainOverview() {data-testid="overview-destination-new"type="button"aria-pressed={previewDestination === "new"}+ disabled={dashboardLimitReached}className={`border p-3 text-left transition-colors ${previewDestination === "new"? "border-foreground bg-muted/40": "border-border hover:border-foreground"- }`}+ } disabled:cursor-not-allowed disabled:opacity-50`}onClick={() => setPreviewDestination("new")}><span className="block text-xs font-medium">create new</span><span className="mt-1 block text-[11px] text-muted-foreground">- keep “{view?.title}” unchanged+ {dashboardLimitReached+ ? `${MAX_DASHBOARDS} dashboard limit reached`+ : `keep “${view?.title}” unchanged`}</span></button><button@@ -1452,6 +1486,16 @@ export function BrainOverview() {</div>)}+ {wholeDashboardPreview && dashboardLimitReached && (+ <div+ data-testid="overview-dashboard-limit-warning"+ className="mb-5 border border-border bg-muted/30 px-4 py-3 text-xs text-muted-foreground"+ >+ You already have {MAX_DASHBOARDS} dashboards. Replace the current+ dashboard, or delete one before creating another.+ </div>+ )}+<div className="grid grid-cols-12 gap-4">{previewSlots.map((slot) => (<OverviewCard@@ -1617,7 +1661,12 @@ export function BrainOverview() {</div>)}<div className="mb-5">- <LiveViewAiComposer busy={generating} compact onGenerate={generate} />+ <LiveViewAiComposer+ busy={generating}+ compact+ currentViewTitle={view.title}+ onGenerate={generate}+ /></div>{undoView && (<divdiff --git a/apps/screenpipe-app-tauri/components/settings/live-view-ai-composer.tsx b/apps/screenpipe-app-tauri/components/settings/live-view-ai-composer.tsxindex 96b433b4d..a4a0ed28f 100644--- a/apps/screenpipe-app-tauri/components/settings/live-view-ai-composer.tsx+++ b/apps/screenpipe-app-tauri/components/settings/live-view-ai-composer.tsx@@ -28,16 +28,55 @@ const GENERATION_STEPS = [type LiveViewAiComposerProps = {busy: boolean;compact?: boolean;+ currentViewTitle?: string | null;onGenerate: (prompt: string,scope: LiveViewGenerationScope,preset: AIPreset,+ intent: LiveViewGenerationIntent,) => void | Promise<void>;};+export type LiveViewGenerationIntent =+ "new-dashboard" | "add-section" | "replace-dashboard";++const REPLACE_DASHBOARD_PATTERNS = [+ /\b(replace|rebuild|redo|reset)\b/i,+ /\bstart\s+over\b/i,+ /^\s*(remove|delete|drop)\b/i,+ /\b(change|update|edit|simplify|reorganize|reorder|remove|delete|drop)\b.*\b(this|current|my|dashboard|view|overview|section|block|card|chart|metric)\b/i,+ /\bmake\b.*\b(this|current|my)\b.*\b(dashboard|view|overview)?\b/i,+];++const NEW_DASHBOARD_PATTERN =+ /\b(add|create|build|make|generate|new|another)\b.{0,32}\b(dashboard|overview|view)\b/i;++const ADD_SECTION_PATTERNS = [+ /^\s*(add|include)\b/i,+ /\b(also\s+(show|include|track)|show\s+.+\s+too|include\s+.+\s+too)\b/i,+ /\b(a|one|another|new)\s+(section|block|card|chart|metric|table|timeline)\b/i,+];++export function inferLiveViewGenerationIntent(+ prompt: string,+ hasCurrentView: boolean,+): LiveViewGenerationIntent {+ if (!hasCurrentView) return "new-dashboard";+ const request = prompt.trim();+ if (REPLACE_DASHBOARD_PATTERNS.some((pattern) => pattern.test(request))) {+ return "replace-dashboard";+ }+ if (NEW_DASHBOARD_PATTERN.test(request)) return "new-dashboard";+ if (ADD_SECTION_PATTERNS.some((pattern) => pattern.test(request))) {+ return "add-section";+ }+ return "new-dashboard";+}+export function LiveViewAiComposer({busy,compact = false,+ currentViewTitle = null,onGenerate,}: LiveViewAiComposerProps) {const { settings } = useSettings();@@ -46,7 +85,6 @@ export function LiveViewAiComposer({[settings.aiPresets],);const [prompt, setPrompt] = useState("");- const [scope, setScope] = useState<LiveViewGenerationScope>("dashboard");const [presetId, setPresetId] = useState<string | null>(null);const [generationStep, setGenerationStep] = useState(0);@@ -71,11 +109,23 @@ export function LiveViewAiComposer({const selectedPreset = presets.find((preset) => preset.id === presetId);const canSubmit = Boolean(prompt.trim() && selectedPreset && !busy);+ const intent = inferLiveViewGenerationIntent(+ prompt,+ Boolean(currentViewTitle),+ );+ const scope: LiveViewGenerationScope =+ intent === "add-section" ? "block" : "dashboard";+ const intentLabel =+ intent === "add-section"+ ? `will add one section to “${currentViewTitle}”`+ : intent === "replace-dashboard"+ ? `will preview changes to “${currentViewTitle}”`+ : "will create a new dashboard";const submit = () => {if (!canSubmit || !selectedPreset) return;setGenerationStep(0);- void onGenerate(prompt.trim(), scope, selectedPreset);+ void onGenerate(prompt.trim(), scope, selectedPreset, intent);};return (@@ -104,7 +154,7 @@ export function LiveViewAiComposer({className="min-h-16 resize-none rounded-none border-0 px-4 py-3 text-sm shadow-none focus-visible:ring-0"placeholder={compact- ? "Ask AI to change this view or add a chart..."+ ? "Ask AI to create a dashboard or add something to this one...": "For example: show how I spend my time and what changed this week"}onChange={(event) => setPrompt(event.target.value)}@@ -117,18 +167,6 @@ export function LiveViewAiComposer({/><div className="flex flex-wrap items-center justify-between gap-2 border-t border-border px-2 py-2"><div className="flex min-w-0 flex-wrap items-center gap-2">- <select- aria-label="generation scope"- value={scope}- disabled={busy}- className="h-8 border border-border bg-background px-2 text-xs outline-none focus:border-foreground"- onChange={(event) =>- setScope(event.target.value as LiveViewGenerationScope)- }- >- <option value="dashboard">whole dashboard</option>- <option value="block">one section</option>- </select><AIPresetsSelectorcontrolledPresetId={presetId}onControlledSelect={setPresetId}@@ -138,6 +176,14 @@ export function LiveViewAiComposer({containerClassName="w-auto min-w-36"triggerClassName="h-8 rounded-none"/>+ {prompt.trim() && (+ <span+ data-testid="live-view-generation-intent"+ className="max-w-72 truncate text-[11px] text-muted-foreground"+ >+ {intentLabel}+ </span>+ )}</div><Buttondata-testid="live-view-ai-generate"diff --git a/apps/screenpipe-app-tauri/lib/live-views/generate-live-view-with-pi.ts b/apps/screenpipe-app-tauri/lib/live-views/generate-live-view-with-pi.tsindex 333996288..4bfd0ebeb 100644--- a/apps/screenpipe-app-tauri/lib/live-views/generate-live-view-with-pi.ts+++ b/apps/screenpipe-app-tauri/lib/live-views/generate-live-view-with-pi.ts@@ -217,7 +217,7 @@ function generationPrompt(options: GenerateLiveViewOptions): string {const scopeInstruction =options.scope === "block"? "Create exactly one new section to add to the existing Live View."- : "Create a complete Live View with 4 to 7 useful, visually varied sections. Return the full replacement dashboard.";+ : "Create a complete Live View with 4 to 7 useful, visually varied sections. Return the full dashboard.";const pipes = relevantPipes(options.prompt, options.pipes).map((pipe) => ({name: pipe.name,description: pipe.description.slice(0, 500),