YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
feat(workflows): discuss open questions through workflow feedback
louise83dee1a3b61
diff --git a/apps/screenpipe-app-tauri/components/workflows/workflow-feedback.test.tsx b/apps/screenpipe-app-tauri/components/workflows/workflow-feedback.test.tsxindex 06312b87c..4a299805a 100644--- a/apps/screenpipe-app-tauri/components/workflows/workflow-feedback.test.tsx+++ b/apps/screenpipe-app-tauri/components/workflows/workflow-feedback.test.tsx@@ -24,6 +24,29 @@ function submit() {}describe("workflow feedback conversation", () => {+ it("opens a selected question without an AI call and keeps its answer scoped after navigation", async () => {+ const saved = emptyAssistantState(); saved.conversations[0].draft = "Keep my draft";+ const { platform, props, rerender } = setup({ load: vi.fn().mockResolvedValue(saved) });+ const question = "Who reviews the synthesis before it is shared?";+ act(() => { window.dispatchEvent(new CustomEvent("workflows:feedback", { detail: { ...feedback, question } })); });+ await screen.findByText(question);+ expect(platform.ask).not.toHaveBeenCalled();+ expect(platform.saveFeedback).not.toHaveBeenCalled();+ expect(screen.queryByRole("button", { name: "Save feedback" })).not.toBeInTheDocument();+ rerender(<WorkflowAssistant {...props} context={{ key: "different", title: "Another workflow" }} />);+ submit();+ await screen.findByRole("button", { name: "Save feedback" });+ expect(platform.ask).toHaveBeenCalledTimes(1);+ expect(platform.ask).toHaveBeenLastCalledWith(expect.objectContaining({+ history: [expect.objectContaining({ role: "assistant", text: question })],+ context: expect.objectContaining({ workflow: expect.objectContaining({ id: "wf-original" }) }),+ }));+ fireEvent.click(screen.getByRole("button", { name: "Save feedback" }));+ await screen.findByText("Feedback saved for the next update");+ expect(platform.saveFeedback).toHaveBeenCalledWith(expect.objectContaining({ id: "wf-original" }), expect.stringContaining(question));+ expect(vi.mocked(platform.save).mock.calls.at(-1)![0].conversations[0].draft).toBe("Keep my draft");+ });+it("automatically reviews a fresh chat on every click and preserves ordinary drafts", async () => {const saved = emptyAssistantState(); saved.conversations[0].draft = "An unfinished question";const { platform } = setup({ load: vi.fn().mockResolvedValue(saved) });diff --git a/apps/screenpipe-workflows-web/scripts/eval-workflow-editor.cjs b/apps/screenpipe-workflows-web/scripts/eval-workflow-editor.cjsindex eac3d89ec..289290dd5 100644--- a/apps/screenpipe-workflows-web/scripts/eval-workflow-editor.cjs+++ b/apps/screenpipe-workflows-web/scripts/eval-workflow-editor.cjs@@ -26,6 +26,31 @@ require("node:fs").mkdirSync(out, { recursive: true });assert.equal(await button("Save").count(), 0);assert.equal(await button("Move step 1").getAttribute("draggable"), "true");});+ await check("open questions launch focused feedback and preserve unanswered questions", async () => {+ const question = "Who reviews the synthesis before it is shared?";+ const questions = page.getByRole("region", { name: "Open questions", exact: true });+ await questions.getByRole("button", { name: question, exact: true }).focus();+ await page.keyboard.press("Enter");+ const chat = page.getByRole("region", { name: "Screenpipe assistant", exact: true });+ await chat.getByText(question, { exact: true }).waitFor();+ assert.equal(await chat.getByRole("button", { name: "Stop answer" }).count(), 0);+ assert.equal(await chat.getByRole("button", { name: "Save feedback" }).count(), 0);+ await page.screenshot({ path: out + "/question-chat.png" });+ await chat.getByRole("textbox", { name: "Ask Screenpipe", exact: true }).fill("The research lead reviews it before sharing.");+ await chat.getByRole("button", { name: "Send feedback", exact: true }).click();+ await chat.getByRole("button", { name: "Save feedback", exact: true }).click();+ await chat.getByText("Feedback saved for the next update", { exact: true }).waitFor();+ await chat.getByRole("button", { name: "Minimize chat", exact: true }).click();+ assert(await questions.getByRole("button", { name: question, exact: true }).isVisible());+ const references = page.getByRole("region", { name: "References for Collect sources", exact: true });+ await references.getByText("Unresolved details", { exact: true }).click();+ await references.getByRole("button", { name: "Where should the source list be saved?", exact: true }).click();+ await chat.getByText("Where should the source list be saved?", { exact: true }).waitFor();+ await chat.getByText("Collect sources", { exact: true }).waitFor();+ await chat.getByRole("button", { name: "Minimize chat", exact: true }).click();+ await references.getByText("Unresolved details", { exact: true }).click();+ await field("Workflow title").focus();+ });await check("details stay visible while editing controls disclose on hover and focus", async () => {initialBlocks = await page.getByRole("article", { name: "Step 1", exact: true }).getByRole("textbox").evaluateAll(es => es.filter(e => e.getAttribute("aria-label").startsWith("Block ")).map(e => e.value));assert.equal(initialBlocks.length, 3);diff --git a/packages/workflows-ui/src/fixture-platform.ts b/packages/workflows-ui/src/fixture-platform.tsindex 8d3993f95..f85c10cc1 100644--- a/packages/workflows-ui/src/fixture-platform.ts+++ b/packages/workflows-ui/src/fixture-platform.ts@@ -206,6 +206,10 @@ const workflows = [], activeMinutes: 51, waitingMinutes: 7, friction: "Frequent tab switching breaks focus", control: "direct" }, 4),];+// Fictional uncertainties exercise the same clarification flow as saved workflows.+workflows[4].openQuestions = ["Who reviews the synthesis before it is shared?", "What makes a source reliable enough to include?"];+workflows[4].stages[0].openQuestions = ["Where should the source list be saved?"];+const totalMinutes = 1_860;export const fixtureWorkflowAnalysis: WorkflowAnalysis = {diff --git a/packages/workflows-ui/src/workflow-assistant.tsx b/packages/workflows-ui/src/workflow-assistant.tsxindex 2cd3a03a1..a6ba1d210 100644--- a/packages/workflows-ui/src/workflow-assistant.tsx+++ b/packages/workflows-ui/src/workflow-assistant.tsx@@ -39,8 +39,8 @@ export function WorkflowAssistant({ platform, context, onDockChange, onWidthChanconst [historyOpen, setHistoryOpen] = useState(false);const [historyQuery, setHistoryQuery] = useState("");const [displayOpen, setDisplayOpen] = useState(false);- const [pendingFeedback, setPendingFeedback] = useState<AssistantContext | null>(null);- const consumedFeedback = useRef<AssistantContext | null>(null);+ const [pendingFeedback, setPendingFeedback] = useState<{ context: AssistantContext; question?: string } | null>(null);+ const consumedFeedback = useRef<typeof pendingFeedback>(null);const savingFeedback = useRef(false);const [feedbackSaving, setFeedbackSaving] = useState(false);const [feedbackError, setFeedbackError] = useState("");@@ -168,12 +168,12 @@ export function WorkflowAssistant({ platform, context, onDockChange, onWidthChanuseEffect(() => {if (!active || !platform.saveFeedback) return;const openFeedback = (event: Event) => {- const selected = (event as CustomEvent<AssistantContext>).detail;+ const selected = (event as CustomEvent<AssistantContext & { question?: string }>).detail;if (!selected?.workflow || selected.purpose !== "feedback") return;const snapshot = assistantContextSnapshot(selected);// Preserve previous corrections locally while keeping media out of chat history.snapshot.workflow!.userCorrection = selected.workflow.userCorrection;- setPendingFeedback(snapshot);+ setPendingFeedback({ context: snapshot, question: typeof selected.question === "string" ? selected.question.trim() : undefined });setHistoryOpen(false);setOpen(true);};@@ -184,13 +184,17 @@ export function WorkflowAssistant({ platform, context, onDockChange, onWidthChanuseEffect(() => {if (!pendingFeedback || !loaded || busy || controller.current || consumedFeedback.current === pendingFeedback) return;consumedFeedback.current = pendingFeedback;- const fresh = { ...newAssistantConversation(), feedbackContext: pendingFeedback };+ const fresh = { ...newAssistantConversation(), feedbackContext: pendingFeedback.context };+ if (pendingFeedback.question) {+ fresh.title = pendingFeedback.question.slice(0, 64);+ fresh.messages = [{ id: crypto.randomUUID(), role: "assistant", text: pendingFeedback.question, at: new Date().toISOString() }];+ }update(current => ({ ...current, activeId: fresh.id, conversations: [...current.conversations, fresh] }));setPendingFeedback(null);setIncludeContext(true);setError("");setFeedbackError("");- void send(FEEDBACK_PROMPT);+ if (!pendingFeedback.question) void send(FEEDBACK_PROMPT);requestAnimationFrame(() => input.current?.focus());}, [pendingFeedback, loaded, busy, update]);diff --git a/packages/workflows-ui/src/workflow-question.module.css b/packages/workflows-ui/src/workflow-question.module.cssnew file mode 100644index 000000000..78dec3f18--- /dev/null+++ b/packages/workflows-ui/src/workflow-question.module.css@@ -0,0 +1,7 @@+/* screenpipe — AI that knows everything you've seen, said, or heard */+/* https://screenpipe.com */+.question { display: flex; align-items: flex-start; gap: 9px; width: 100%; padding: 9px 6px; border: 0; border-radius: 4px; background: transparent; color: var(--ink, #171914); font: inherit; font-size: 13px; line-height: 1.5; text-align: left; cursor: pointer; }+.question span { flex: 1; min-width: 0; overflow-wrap: anywhere; }+.question svg { flex-shrink: 0; margin-top: 3px; color: var(--muted, #78786f); }+.question:hover { background: var(--soft, #f1f2ed); }+.question:focus-visible { outline: 1px solid currentColor; outline-offset: 2px; }diff --git a/packages/workflows-ui/src/workflow-question.tsx b/packages/workflows-ui/src/workflow-question.tsxnew file mode 100644index 000000000..2e1547ce8--- /dev/null+++ b/packages/workflows-ui/src/workflow-question.tsx@@ -0,0 +1,19 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+"use client";+import { MessageCircle, ArrowUpRight } from "lucide-react";+import type { WorkflowMap } from "./model";+import styles from "./workflow-question.module.css";++export function WorkflowQuestion({ workflow, question, stage, interactive }: {+ workflow: WorkflowMap; question: string; stage?: string; interactive: boolean;+}) {+ if (!interactive) return <span>{question}</span>;+ return <button type="button" className={styles.question} title="Discuss in chat"+ onClick={() => window.dispatchEvent(new CustomEvent("workflows:feedback", { detail: {+ key: `feedback:${workflow.id || workflow.title}`, title: workflow.title,+ workflow, purpose: "feedback", question: stage ? `${stage}\n\n${question}` : question,+ } }))}>+ <MessageCircle size={14} aria-hidden="true" /><span>{question}</span><ArrowUpRight size={14} aria-hidden="true" />+ </button>;+}diff --git a/packages/workflows-ui/src/workflow-step-evidence.tsx b/packages/workflows-ui/src/workflow-step-evidence.tsxindex b23204c5e..c72532b26 100644--- a/packages/workflows-ui/src/workflow-step-evidence.tsx+++ b/packages/workflows-ui/src/workflow-step-evidence.tsx@@ -6,6 +6,7 @@ import { useMemo, useState } from "react";import { ChevronDown, Maximize2, Minimize2, Play, X } from "lucide-react";import type { WorkflowMap, WorkflowStage } from "./model";import type { WorkflowsPlatform } from "./platform";+import { WorkflowQuestion } from "./workflow-question";import { WorkflowReplay } from "./workflow-replay";import styles from "./workflow-step-evidence.module.css";@@ -70,6 +71,6 @@ export function WorkflowStepEvidence({ workflow, stage, platform }: {</div></details>}</div>- {!!stage.openQuestions?.length && <details className={styles.questions}><summary>Unresolved details<ChevronDown size={12} /></summary><ul>{stage.openQuestions.map(q => <li key={q}>{q}</li>)}</ul></details>}+ {!!stage.openQuestions?.length && <details className={styles.questions}><summary>Unresolved details<ChevronDown size={12} /></summary><ul>{stage.openQuestions.map(q => <li key={q}><WorkflowQuestion workflow={workflow} question={q} stage={stage.name} interactive={!!platform.assistant?.saveFeedback} /></li>)}</ul></details>}</section>;}diff --git a/packages/workflows-ui/src/workflows-app.tsx b/packages/workflows-ui/src/workflows-app.tsxindex 5aa84c530..168c26eee 100644--- a/packages/workflows-ui/src/workflows-app.tsx+++ b/packages/workflows-ui/src/workflows-app.tsx@@ -4,6 +4,7 @@"use client";import { useNavigationWidth } from "./use-navigation-width";+import { WorkflowQuestion } from "./workflow-question";import { WorkflowStepEvidence } from "./workflow-step-evidence";import { WorkflowEditor } from "./workflow-editor";import { retainNewerWorkflowEdits, type WorkflowEdit } from "./workflow-edits";@@ -978,7 +979,7 @@ function WorkflowDetail({ workflow, navigate, platform, workProfile, saveCorrect<p>{detail.text}</p>{detail.quote && <details><summary>{detail.userEdited || stage.userEdited ? "Original reference" : "Source excerpt"} · {detail.app}</summary><blockquote>{detail.quote}</blockquote><small>{formatEvidenceTimestamp(detail.timestamp)} · {detail.userEdited || stage.userEdited ? "Edited instructions are not verified by this reference" : "Text match, not execution verification"}</small></details>}</li>)}</ol> : <p>Step not yet verified.</p>}- {!!stage.openQuestions?.length && <details className={styles.procedureQuestions}><summary>Unresolved details</summary><ul>{stage.openQuestions.map((question) => <li key={question}>{question}</li>)}</ul></details>}+ {!!stage.openQuestions?.length && <details className={styles.procedureQuestions}><summary>Unresolved details</summary><ul>{stage.openQuestions.map((question) => <li key={question}><WorkflowQuestion workflow={workflow} question={question} stage={stage.name} interactive={!!platform.assistant?.saveFeedback} /></li>)}</ul></details>}</section>{stage.screenshot?.visualVerified && <div className={styles.stageScreenshot}><>@@ -992,6 +993,7 @@ function WorkflowDetail({ workflow, navigate, platform, workProfile, saveCorrect</div><div className={styles.flowEndpoint}><span>Ends with</span><strong>{workflow.outcome}</strong></div></section>}+ {!!workflow.openQuestions?.length && <section aria-label="Open questions"><h2>Open questions</h2><div>{workflow.openQuestions.map((question, index) => <WorkflowQuestion key={`${index}:${question}`} workflow={workflow} question={question} interactive={!!platform.assistant?.saveFeedback} />)}</div></section>}{!!workflow.variations.length && <section className={styles.panel}><h2>Variations</h2><ul className={styles.plainList}>{workflow.variations.map(item => <li key={item}>{item}</li>)}</ul></section>}{!!actionableFriction.length && <section><div className={styles.sectionHeading}><div><h2>Friction you can affect</h2></div></div><BottleneckList items={actionableFriction.map((item) => ({ ...item, workflowTitle: workflow.title, repetitions: workflow.repetitions }))} /></section>}{!!constraints.length && <section><div className={styles.sectionHeading}><div><h2>External and required constraints</h2></div></div><BottleneckList items={constraints.map((item) => ({ ...item, workflowTitle: workflow.title, repetitions: workflow.repetitions }))} numbered={false} /></section>}@@ -1008,7 +1010,6 @@ function WorkflowDetail({ workflow, navigate, platform, workProfile, saveCorrect<p className={styles.panelEmpty}>References show where text was captured. They do not prove task completion or a continuous sequence. Model confidence is not an accuracy score.</p>{!!workflow.captureSequence?.length && <section aria-label="Ordered capture example"><strong>Ordered capture example</strong><p className={styles.panelEmpty}>Check that these moments concern the same task. Time order alone does not establish this.</p><ol>{workflow.captureSequence.map((entry, index) => <li key={`${entry.timestamp}-${index}`}><details><summary>{workflow.stages[index]?.name} · {formatEvidenceTimestamp(entry.timestamp)} · {entry.app}</summary><p>{entry.detail}</p></details></li>)}</ol></section>}{timing && <section aria-label="Time per run"><strong>Time per run</strong><p className={styles.panelEmpty}>{formatMinutes(timing.minMinutes)}–{formatMinutes(timing.maxMinutes)} across {timing.sampleCount} run{timing.sampleCount === 1 ? "" : "s"}. Estimated elapsed time includes pauses; it is not active work time.</p><ol>{timing.runs.map(run => <li key={run.start.timestamp}><details><summary>{formatEvidenceTimestamp(run.start.timestamp)} · {formatMinutes((Date.parse(run.end.timestamp) - Date.parse(run.start.timestamp)) / 60_000)}</summary><p>{run.summary}</p>{(["start", "end"] as const).map(boundary => <div key={boundary}><strong>{boundary === "start" ? "Started" : "Finished"}</strong><p>{formatEvidenceTimestamp(run[boundary].timestamp)} · {run[boundary].app}</p><blockquote>{run[boundary].quote}</blockquote><TimingSourceButton timestamp={run[boundary].timestamp} open={platform.assistant?.openLink} /></div>)}</details></li>)}</ol></section>}- {!!workflow.openQuestions?.length && <section aria-label="Open questions"><strong>Open questions</strong><ul>{workflow.openQuestions.map(question => <li key={question}>{question}</li>)}</ul></section>}{!!workflow.limitations?.length && <ul>{workflow.limitations.map((limitation) => <li key={limitation}>{limitation}</li>)}</ul>}</details></div>