YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(sops): use the existing companion chat for edits
louisfe51bf3becdf
diff --git a/apps/screenpipe-app-tauri/lib/workflows/guide.test.tsx b/apps/screenpipe-app-tauri/lib/workflows/guide.test.tsxindex 2eca9c9b6..87798bbe4 100644--- a/apps/screenpipe-app-tauri/lib/workflows/guide.test.tsx+++ b/apps/screenpipe-app-tauri/lib/workflows/guide.test.tsx@@ -1,6 +1,6 @@// screenpipe — AI that knows everything you've seen, said, or heard// https://screenpipe.com-import React from "react";+import React, { useState } from "react";import { describe, it, expect, vi, afterEach } from "vitest";import {render,@@ -18,6 +18,20 @@ import {import { WorkflowGuide } from "../../../../packages/workflows-ui/src/workflow-guide";import { fixtureWorkflowAnalysis } from "../../../../packages/workflows-ui/src/fixture-platform";import { sanitizeWorkflowAnalysis } from "../../../../packages/workflows-ui/src/catalog";+import { PageAssistantContext, type PageAssistant } from "../../../../packages/workflows-ui/src/page-assistant";+import { WorkflowAssistant } from "../../../../packages/workflows-ui/src/workflow-assistant";+import { emptyAssistantState } from "../../../../packages/workflows-ui/src/assistant";+import type { WorkflowsPlatform } from "../../../../packages/workflows-ui/src/platform";+function GuideChat({ edit, save = vi.fn(), previousFeedback = false }: { edit: NonNullable<WorkflowsPlatform["guides"]>["edit"]; save?: ReturnType<typeof vi.fn>; previousFeedback?: boolean }) {+ const previous = emptyAssistantState();+ previous.conversations[0].feedbackContext = { key: "feedback:research", title: workflow.title, workflow, purpose: "feedback" };+ previous.conversations[0].draft = "Keep this feedback draft";+ const [page, setPage] = useState<PageAssistant | null>(null);+ return <PageAssistantContext.Provider value={setPage}>+ <WorkflowGuide workflow={workflow} close={() => {}} platform={{ load: async () => guide, generate: vi.fn(), save, export: vi.fn(), edit }} />+ {page && <WorkflowAssistant context={page.context} onDockChange={() => {}} platform={{ load: async () => previousFeedback ? previous : null, save: async () => {}, ask: page.ask }} />}+ </PageAssistantContext.Provider>;+}const workflow = structuredClone(fixtureWorkflowAnalysis.analysis.workflows[0]);workflow.id = "research";workflow.revision = 3;@@ -354,27 +368,30 @@ describe("SOP assistant and web editor", () => {signal = s;return new Promise<Guide>(() => {});});- render(- <WorkflowGuide- workflow={workflow}- close={() => {}}- platform={{- load: async () => guide,- generate: vi.fn(),- save: vi.fn(),- export: vi.fn(),- edit,- }}- />,- );- const input = await screen.findByLabelText(- "Ask Screenpipe to edit the SOP",- );- fireEvent.change(input, { target: { value: "Make it shorter" } });- fireEvent.click(screen.getByLabelText("Edit with Screenpipe"));+ render(<GuideChat edit={edit} />);+ expect(screen.queryByRole("region", { name: "SOP assistant" })).not.toBeInTheDocument();+ fireEvent.click(await screen.findByRole("button", { name: "Ask Screenpipe" }));+ fireEvent.change(await screen.findByRole("textbox", { name: "Ask Screenpipe" }), { target: { value: "Make it shorter" } });+ await waitFor(() => expect(screen.getByLabelText("Send message")).toBeEnabled());+ fireEvent.click(screen.getByLabelText("Send message"));+ await waitFor(() => expect(edit).toHaveBeenCalled());expect(edit.mock.calls[0][2]).toBe("Make it shorter");- fireEvent.click(screen.getByText("Stop"));+ fireEvent.click(screen.getByLabelText("Stop answer"));expect(signal?.aborted).toBe(true);- expect(screen.getByText("Stopped. Your SOP is saved.")).toBeInTheDocument();});+ it("applies a chat edit to the SOP and saves it", async () => {+ const save = vi.fn();+ const edit = vi.fn(async (draft: Guide) => ({ ...draft, title: "Shorter research SOP" }));+ render(<GuideChat edit={edit} save={save} previousFeedback />);+ fireEvent.click(await screen.findByRole("button", { name: "Ask Screenpipe" }));+ fireEvent.change(await screen.findByRole("textbox", { name: "Ask Screenpipe" }), { target: { value: "Shorten the title" } });+ await waitFor(() => expect(screen.getByLabelText("Send message")).toBeEnabled());+ fireEvent.click(screen.getByLabelText("Send message"));+ expect(await screen.findByRole("heading", { name: "Shorter research SOP" })).toBeInTheDocument();+ await waitFor(() => expect(save).toHaveBeenCalledWith(expect.objectContaining({ title: "Shorter research SOP" })));+ fireEvent.click(screen.getByLabelText("Minimize chat"));+ fireEvent.click(screen.getByRole("button", { name: "Ask Screenpipe" }));+ expect(screen.getAllByText("Shorten the title").length).toBeGreaterThan(0);+ });+});diff --git a/packages/workflows-ui/src/assistant.ts b/packages/workflows-ui/src/assistant.tsindex ceb3c8897..64cbe4e36 100644--- a/packages/workflows-ui/src/assistant.ts+++ b/packages/workflows-ui/src/assistant.ts@@ -4,7 +4,7 @@import type { WorkProfile, WorkflowMap } from "./model";export type AssistantContext = {- purpose?: "feedback";+ purpose?: "feedback" | "sop";key: string;title: string;workflow?: WorkflowMap;@@ -20,7 +20,7 @@ export type AssistantMessage = {status?: "stopped" | "error";feedbackSaved?: boolean;};-export type AssistantConversation = { id: string; title: string; messages: AssistantMessage[]; draft: string; feedbackContext?: AssistantContext };+export type AssistantConversation = { id: string; title: string; messages: AssistantMessage[]; draft: string; pageKey?: string; feedbackContext?: AssistantContext };export type AssistantState = {version: 1;mode: "floating" | "sidebar";diff --git a/packages/workflows-ui/src/guide-assistant.tsx b/packages/workflows-ui/src/guide-assistant.tsxindex a518a58d1..836326ab5 100644--- a/packages/workflows-ui/src/guide-assistant.tsx+++ b/packages/workflows-ui/src/guide-assistant.tsx@@ -1,117 +1,53 @@// screenpipe — AI that knows everything you've seen, said, or heard// https://screenpipe.com"use client";-import { useEffect, useRef, useState } from "react";-import { ArrowUp, Loader2, X } from "lucide-react";+import { useContext, useEffect, useRef } from "react";import type { WorkflowGuide } from "./guide";import type { WorkflowMap } from "./model";import type { WorkflowsPlatform } from "./platform";-import styles from "./workflow-guide.module.css";-export function GuideAssistant({- guide,- workflow,- platform,- update,-}: {+import { PageAssistantContext } from "./page-assistant";++/** Routes SOP edits through the shell's chat, including history, dictation and stop. */+export function GuideAssistant(props: {guide: WorkflowGuide;workflow: WorkflowMap;platform: NonNullable<WorkflowsPlatform["guides"]>;update: (guide: WorkflowGuide) => void;}) {- const [request, setRequest] = useState("");- const [busy, setBusy] = useState(false);- const [progress, setProgress] = useState("");- const [error, setError] = useState("");- const controller = useRef<AbortController>();- const currentGuide = useRef(guide);- currentGuide.current = guide;- useEffect(- () => () => {- controller.current?.abort();- },- [],- );- async function run() {- const run = new AbortController();- controller.current = run;- setBusy(true);- setError("");- setProgress("Editing your SOP");- const snapshot = guide;- const report = (message: string) => {- if (!run.signal.aborted) setProgress(message);- };- try {- if (platform.edit) {- const next = await platform.edit(- snapshot,- workflow,- request,- run.signal,- report,- );- if (run.signal.aborted) return;- if (currentGuide.current !== snapshot)- throw new Error(- "The SOP changed while the assistant was editing. Your edits were kept. Try again.",- );- update(next);- setRequest("");- }- } catch (e) {- if (!run.signal.aborted)- setError(- e instanceof Error- ? e.message- : "Could not finish. Your SOP is saved.",- );- } finally {- if (!run.signal.aborted) setBusy(false);- }- }- return (- <section className={styles.assistantPanel} aria-label="SOP assistant">- {platform.edit && (- <form- className={styles.guideComposer}- onSubmit={(e) => {- e.preventDefault();- if (request.trim() && !busy) void run();- }}- >- <input- aria-label="Ask Screenpipe to edit the SOP"- placeholder="Ask Screenpipe to shorten, explain, or rewrite this SOP…"- value={request}- maxLength={4000}- disabled={busy}- onChange={(e) => setRequest(e.target.value)}- />- <button- aria-label="Edit with Screenpipe"- disabled={busy || !request.trim()}- >- <ArrowUp size={17} />- </button>- </form>- )}- {busy && (- <div className={styles.assistantProgress} role="status">- <Loader2 className={styles.spin} size={16} />- <span>{progress}</span>- <button- onClick={() => {- controller.current?.abort();- setBusy(false);- setError("Stopped. Your SOP is saved.");- }}- >- <X size={14} />- Stop- </button>- </div>- )}- {error && <p role="alert">{error}</p>}- </section>- );+ const register = useContext(PageAssistantContext);+ const current = useRef(props);+ current.current = props;+ useEffect(() => {+ if (!register) return;+ const lifetime = new AbortController();+ register({+ context: { key: `sop:${props.workflow.id || props.workflow.title}`, title: `SOP: ${props.guide.title}`, purpose: "sop" },+ ask: async ({ question, signal, onProgress }) => {+ const { guide, workflow, platform, update } = current.current;+ if (!platform.edit) throw new Error("SOP editing is unavailable.");+ const run = new AbortController();+ const abort = () => run.abort();+ signal.addEventListener("abort", abort, { once: true });+ lifetime.signal.addEventListener("abort", abort, { once: true });+ if (signal.aborted || lifetime.signal.aborted) abort();+ const runSignal = run.signal;+ try {+ runSignal.throwIfAborted();+ const next = await platform.edit(guide, workflow, question, runSignal, text => {+ if (!runSignal.aborted) onProgress({ text, activity: "writing" });+ });+ runSignal.throwIfAborted();+ if (current.current.guide !== guide)+ throw new Error("The SOP changed while the assistant was editing. Your edits were kept. Try again.");+ update(next);+ return "Updated the SOP. You can review the changes on the page.";+ } finally {+ signal.removeEventListener("abort", abort);+ lifetime.signal.removeEventListener("abort", abort);+ }+ },+ });+ return () => { lifetime.abort(); register(null); };+ }, [register, props.workflow.id, props.workflow.title]);+ return null;}diff --git a/packages/workflows-ui/src/page-assistant.ts b/packages/workflows-ui/src/page-assistant.tsnew file mode 100644index 000000000..0db49a999--- /dev/null+++ b/packages/workflows-ui/src/page-assistant.ts@@ -0,0 +1,12 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+"use client";+import { createContext, type Dispatch, type SetStateAction } from "react";+import type { AssistantContext, WorkflowsAssistantPlatform } from "./assistant";++/** A mounted page can supply an action to the existing chat without another composer. */+export type PageAssistant = {+ context: AssistantContext;+ ask: WorkflowsAssistantPlatform["ask"];+};+export const PageAssistantContext = createContext<Dispatch<SetStateAction<PageAssistant | null>> | null>(null);diff --git a/packages/workflows-ui/src/workflow-assistant.tsx b/packages/workflows-ui/src/workflow-assistant.tsxindex 800a4ea1e..48f16980a 100644--- a/packages/workflows-ui/src/workflow-assistant.tsx+++ b/packages/workflows-ui/src/workflow-assistant.tsx@@ -62,6 +62,7 @@ export function WorkflowAssistant({ platform, context, onDockChange, onWidthChanconst loadInFlight = useRef(false);const conversation = state.conversations.find((c) => c.id === state.activeId)!;const feedbackContext = conversation.feedbackContext;+ const selectedSop = useRef<string | null>(null);const width = Number.isFinite(state.sidebarWidth) ? Math.max(340, Math.min(560, state.sidebarWidth!)) : 420;const persist = useCallback(async (snapshot: AssistantState) => {@@ -95,6 +96,17 @@ export function WorkflowAssistant({ platform, context, onDockChange, onWidthChan}, [platform]);useEffect(() => { if (open && !loadedRef.current && !loadError) void restore(); }, [open, restore, loadError]);+ useEffect(() => {+ if (context.purpose !== "sop") { selectedSop.current = null; return; }+ if (!loaded || busy || selectedSop.current === context.key) return;+ selectedSop.current = context.key;+ // Keep existing feedback and ordinary drafts separate from SOP edits.+ const existing = [...stateRef.current.conversations].reverse().find(c =>+ !c.feedbackContext && (c.pageKey === context.key || c.messages.some(m => m.context?.key === context.key)));+ const next = existing ?? { ...newAssistantConversation(), pageKey: context.key };+ update(s => ({ ...s, activeId: next.id, conversations: existing ? s.conversations : [...s.conversations, next] }));+ setError(""); setHistoryOpen(false);+ }, [context.key, context.purpose, loaded, busy, update]);useEffect(() => { onDockChange(open && state.mode === "sidebar"); }, [open, state.mode, onDockChange]);useEffect(() => { onWidthChange?.(width); }, [width, onWidthChange]);useEffect(() => { onOpenChange?.(open); }, [open, onOpenChange]);@@ -255,7 +267,9 @@ export function WorkflowAssistant({ platform, context, onDockChange, onWidthChanconst lastUser = [...conversation.messages].reverse().find((message) => message.role === "user");const lastAnswer = conversation.messages.at(-1);- const suggestions = context.workflow+ const suggestions = context.purpose === "sop"+ ? ["Make this SOP shorter", "Make the steps easier to follow"]+ : context.workflow? ["Summarize this workflow", "Find recent examples in my memory"]: ["What did I work on yesterday?", "Find a conversation I had this week"];const history = [...state.conversations].reverse().filter((item) =>@@ -274,7 +288,7 @@ export function WorkflowAssistant({ platform, context, onDockChange, onWidthChanrequestAnimationFrame(() => input.current?.focus());}function newConversation() {- const fresh = newAssistantConversation();+ const fresh = { ...newAssistantConversation(), ...(context.purpose === "sop" ? { pageKey: context.key } : {}) };update((current) => ({ ...current, activeId: fresh.id, conversations: [...current.conversations, fresh] }));setHistoryOpen(false); setHistoryQuery(""); setError("");requestAnimationFrame(() => input.current?.focus());@@ -334,7 +348,7 @@ export function WorkflowAssistant({ platform, context, onDockChange, onWidthChan{!history.length && <p>{historyQuery ? "No matching conversations." : "Your conversations will appear here."}</p>}</div> : loaded && <>{!conversation.messages.length && <div className={styles.empty}>- <h2>{feedbackContext ? "What should change?" : context.workflow ? "Ask about this workflow" : "Search your memory"}</h2>+ <h2>{feedbackContext ? "What should change?" : context.purpose === "sop" ? "Edit this SOP" : context.workflow ? "Ask about this workflow" : "Search your memory"}</h2>{!feedbackContext && <div>{suggestions.map((question) => <button key={question} onClick={() => void send(question)}><Search size={15} /><span>{question}</span><ArrowUp size={13} /></button>)}</div>}</div>}{conversation.messages.map((message) => <article key={message.id} className={message.role === "user" ? styles.user : styles.assistant} aria-label={message.role === "user" ? "Your question" : "Screenpipe answer"}>@@ -360,7 +374,7 @@ export function WorkflowAssistant({ platform, context, onDockChange, onWidthChan{saveError && <div className={styles.saveError} role="alert">Couldn’t save this conversation.<button onClick={() => void persist(stateRef.current).catch(() => {})}>Retry save</button></div>}{!historyOpen && <form className={styles.composer} onSubmit={(event) => { event.preventDefault(); void send(conversation.draft); }}>{feedbackContext ? <span className={styles.context}><span className={styles.contextDot} /><span>{feedbackContext.title}</span></span> : <button type="button" className={styles.context} aria-pressed={includeContext} title={includeContext ? "Remove current page from the next message" : "Include current page in the next message"} onClick={() => setIncludeContext(!includeContext)}>{includeContext ? <><span className={styles.contextDot} /><span>{context.title}</span><X size={12} /></> : <><Plus size={13} /><span>Add current page</span></>}</button>}- <ComposerTextArea ref={input} aria-label="Ask Screenpipe" placeholder={feedbackContext ? platform.learnsFromFeedback ? "Share feedback to refine this workflow…" : "Answer a question or share feedback…" : includeContext && context.workflow ? "Ask about this workflow…" : "Ask or find anything…"} rows={1}+ <ComposerTextArea ref={input} aria-label="Ask Screenpipe" placeholder={feedbackContext ? platform.learnsFromFeedback ? "Share feedback to refine this workflow…" : "Answer a question or share feedback…" : includeContext && context.purpose === "sop" ? "Ask Screenpipe to edit this SOP…" : includeContext && context.workflow ? "Ask about this workflow…" : "Ask or find anything…"} rows={1}value={conversation.draft} maxLength={8000} disabled={!loaded} onChange={(event) => update((current) => ({...current, conversations: current.conversations.map((item) => item.id === current.activeId ? { ...item, draft: event.target.value } : item),}))} onSend={() => void send(conversation.draft)} />diff --git a/packages/workflows-ui/src/workflow-guide.module.css b/packages/workflows-ui/src/workflow-guide.module.cssindex 0ad13c25a..d637359f8 100644--- a/packages/workflows-ui/src/workflow-guide.module.css+++ b/packages/workflows-ui/src/workflow-guide.module.css@@ -435,33 +435,3 @@ label.result {margin: 40px auto;}}-.assistantPanel {- margin: 24px 0;- padding: 20px;- background: var(--panel, #fff);- border: 1px solid var(--line, #dfe1d9);- border-radius: 10px;-}-.guideComposer {- display: flex;- gap: 8px;- margin-bottom: 14px;-}-.guideComposer input {- flex: 1;- min-width: 0;- padding: 10px;- border: 0;- background: transparent;- color: inherit;- font: inherit;-}-.assistantProgress {- display: flex;- gap: 10px;- align-items: center;- margin-top: 16px;-}-.assistantProgress span {- flex: 1;-}diff --git a/packages/workflows-ui/src/workflows-app.tsx b/packages/workflows-ui/src/workflows-app.tsxindex 73ac21bb5..57c3a2764 100644--- a/packages/workflows-ui/src/workflows-app.tsx+++ b/packages/workflows-ui/src/workflows-app.tsx@@ -6,6 +6,7 @@import { matchesSidebarShortcut, useSidebarShortcuts } from "./sidebar-shortcuts";import { WorkflowGuide } from "./workflow-guide";import { WorkflowAssistant } from "./workflow-assistant";+import { PageAssistantContext, type PageAssistant } from "./page-assistant";import { WorkflowRunProgress } from "./workflow-run-progress";import { workflowTiming } from "./timing";import { CapturedMomentButton, WorkflowReplay } from "./workflow-replay";@@ -398,6 +399,14 @@ function AppShell({assistant?: { platform: NonNullable<WorkflowsAppProps["platform"]["assistant"]>; context: AssistantContext };children: React.ReactNode;}) {+ const [pageAssistant, setPageAssistant] = useState<PageAssistant | null>(null);+ const assistantPlatform = useMemo(() => assistant && ({+ ...assistant.platform,+ ask: (request: Parameters<typeof assistant.platform.ask>[0]) =>+ pageAssistant && request.context?.key === pageAssistant.context.key+ ? pageAssistant.ask(request)+ : assistant.platform.ask(request),+ }), [assistant?.platform, pageAssistant]);const [assistantDocked, setAssistantDocked] = useState(false);const [assistantOpen, setAssistantOpen] = useState(false);const [assistantMode, setAssistantMode] = useState<AssistantState["mode"]>("sidebar");@@ -450,7 +459,7 @@ function AppShell({</button>;return (- <div data-native-window={nativeMacWindow} data-fullscreen={fullscreen} className={`ph-no-capture ph-mask ${styles.app} ${embedded ? styles.appEmbedded : ""} ${assistantDocked ? styles.appDocked : ""} ${navigationCollapsed ? styles.navigationCollapsed : ""}`} style={{ "--assistant-width": assistantWidth + "px" } as React.CSSProperties}>+ <PageAssistantContext.Provider value={setPageAssistant}><div data-native-window={nativeMacWindow} data-fullscreen={fullscreen} className={`ph-no-capture ph-mask ${styles.app} ${embedded ? styles.appEmbedded : ""} ${assistantDocked ? styles.appDocked : ""} ${navigationCollapsed ? styles.navigationCollapsed : ""}`} style={{ "--assistant-width": assistantWidth + "px" } as React.CSSProperties}>{nativeMacWindow && navigationToggle}<aside id="workflows-navigation" className={styles.sidebar} hidden={navigationCollapsed} aria-label="Navigation sidebar">{navigationBrand ? <div className={styles.integratedBrand}>{navigationBrand}</div> : <div className={styles.brand} data-tauri-drag-region onMouseDown={(event) => handleWindowDrag(event, startWindowDrag)}>@@ -505,8 +514,8 @@ function AppShell({</nav>}<main className={styles.main}>{children}</main></section>- {assistant && <WorkflowAssistant composerAccessory={composerAccessory} active={active} platform={assistant.platform} context={assistant.context} onDockChange={setAssistantDocked} onWidthChange={setAssistantWidth} onOpenChange={setAssistantOpen} onModeChange={setAssistantMode} headerToggle />}- </div>+ {assistant && <WorkflowAssistant composerAccessory={composerAccessory} active={active} platform={assistantPlatform!} context={pageAssistant?.context ?? assistant.context} onDockChange={setAssistantDocked} onWidthChange={setAssistantWidth} onOpenChange={setAssistantOpen} onModeChange={setAssistantMode} headerToggle={!pageAssistant} />}+ </div></PageAssistantContext.Provider>);}