YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(workflows): allow saving answers during voice processing (#7436)
Louis Beaumontdbde4dcb8b72
diff --git a/apps/screenpipe-app-tauri/components/workflows/workflow-questions.test.tsx b/apps/screenpipe-app-tauri/components/workflows/workflow-questions.test.tsxindex 776f7f98c..03520b0bb 100644--- a/apps/screenpipe-app-tauri/components/workflows/workflow-questions.test.tsx+++ b/apps/screenpipe-app-tauri/components/workflows/workflow-questions.test.tsx@@ -1,7 +1,7 @@// screenpipe — AI that knows everything you've seen, said, or heard// https://screenpipe.comimport React from "react";-import { fireEvent, render, screen, waitFor } from "@testing-library/react";+import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";import { describe, expect, it, vi } from "vitest";import { WorkflowQuestions } from "../../../../packages/workflows-ui/src/workflow-questions";import { readWorkflowAnswers, writeWorkflowAnswers } from "../../../../packages/workflows-ui/src/workflow-answers";@@ -82,7 +82,7 @@ it("uses a section voice control, protects manual edits during inference, supporfireEvent.click(screen.getByRole("button", { name: "Stop recording" }));await waitFor(() => expect(voice.fill).toHaveBeenCalled());fireEvent.change(screen.getByLabelText(question), { target: { value: "Typed while filling" } });- expect(screen.getByRole("button", { name: "Save answers" })).toBeDisabled();+ expect(screen.getByRole("button", { name: "Save answers" })).toBeEnabled();resolve([{ question, answer: "The lead", quote: "The lead reviews it." }, { question: "Where is it saved?", answer: "Shared drive", quote: "shared drive" }]);await waitFor(() => expect(screen.getByLabelText("Where is it saved?")).toHaveValue("Shared drive"));expect(screen.getByLabelText(question)).toHaveValue("Typed while filling");@@ -117,3 +117,37 @@ it("does not remount step media when saving answers only changes the revision",expect(screen.getAllByTestId("step-media")[0]).toBe(original);expect(save).not.toHaveBeenCalled();});+++it.each(["recording", "filling"])("saves typed answers during %s and ignores late voice results", async phase => {+ let finish!: (value: any) => void;+ let fillSignal: AbortSignal | undefined;+ const voice: QuestionnaireVoice = { connect: vi.fn(), disconnect: vi.fn(), fill: vi.fn((_input, signal) => {+ fillSignal = signal;+ return new Promise(resolve => { finish = resolve; });+ }) };+ const save = vi.fn(async (userCorrection: string) => ({ ...workflow, userCorrection }));+ render(<WorkflowQuestions workflow={workflow} save={save} voice={voice} />);+ fireEvent.click(screen.getByRole("button", { name: "Answer with voice" }));+ act(() => live.transcript?.("The lead reviews it. Shared drive."));+ if (phase === "filling") {+ fireEvent.click(screen.getByRole("button", { name: "Stop recording" }));+ await waitFor(() => expect(voice.fill).toHaveBeenCalled());+ }+ fireEvent.change(screen.getByLabelText(question), { target: { value: "My reviewed answer" } });+ const button = screen.getByRole("button", { name: "Save answers" });+ expect(button).toBeEnabled();+ fireEvent.click(button);+ await screen.findByText("Answers saved for the next workflow update.");+ expect(save).toHaveBeenCalledTimes(1);+ expect(readWorkflowAnswers(save.mock.calls[0][0])).toEqual([{ question, answer: "My reviewed answer" }]);+ expect(screen.queryByRole("button", { name: "Stop recording" })).not.toBeInTheDocument();+ if (phase === "filling") {+ expect(fillSignal?.aborted).toBe(true);+ await act(async () => finish([{ question: "Where is it saved?", answer: "Late answer", quote: "Shared drive." }]));+ expect(screen.getByLabelText("Where is it saved?")).toHaveValue("");+ }+ expect(button).toBeDisabled();+ fireEvent.click(screen.getByRole("button", { name: "Answer with voice" }));+ expect(screen.getByRole("button", { name: "Stop recording" })).toBeInTheDocument();+});diff --git a/apps/screenpipe-workflows-web/scripts/eval-workflow-questions.cjs b/apps/screenpipe-workflows-web/scripts/eval-workflow-questions.cjsindex d39855666..a5d77b53a 100644--- a/apps/screenpipe-workflows-web/scripts/eval-workflow-questions.cjs+++ b/apps/screenpipe-workflows-web/scripts/eval-workflow-questions.cjs@@ -109,6 +109,37 @@ fs.mkdirSync(out, { recursive: true });await snap("after-saved");await open();assert.equal(await input.inputValue(), "The project lead reviews the brief before it is shared with the team.");+ // Saving manual answers must not wait for a recording or an AI fill.+ for (const phase of ["recording", "filling"]) {+ releaseFill = undefined;+ await questions.getByRole("button", { name: "Answer with voice" }).click();+ await page.getByText("Listening", { exact: true }).waitFor();+ await page.evaluate(() => window.__voiceChannel.onmessage({ data: JSON.stringify({+ type: "session.input_transcript.delta", event_id: "save-test", delta: "The project lead reviews it.",+ }) }));+ if (phase === "filling") {+ await page.getByRole("button", { name: "Stop recording" }).click();+ const deadline = Date.now() + 10000;+ while (!releaseFill && Date.now() < deadline) await page.waitForTimeout(50);+ assert.ok(releaseFill, "Fill request should be in flight");+ }+ const answer = `Reviewed answer during ${phase}`;+ await input.fill(answer);+ const saveButton = page.getByRole("button", { name: "Save answers", exact: true });+ assert.equal(await saveButton.isEnabled(), true);+ await saveButton.scrollIntoViewIfNeeded();+ await snap(`save-during-${phase}`);+ await saveButton.click();+ await page.getByText("Answers saved for the next workflow update.", { exact: true }).waitFor();+ assert.equal(await page.getByRole("button", { name: "Stop recording" }).count(), 0);+ if (releaseFill) releaseFill();+ await open();+ assert.equal(await input.inputValue(), answer);+ }+ // Restore the answer expected by the remaining layout checks.+ await input.fill("The project lead reviews the brief before it is shared with the team.");+ await page.getByRole("button", { name: "Save answers", exact: true }).click();+ await page.getByText("Answers saved for the next workflow update.", { exact: true }).waitFor();const play=page.getByRole("button",{name:/View recording for/}).first();await play.scrollIntoViewIfNeeded();assert.equal((await play.innerText()).trim(), "");@@ -131,6 +162,6 @@ fs.mkdirSync(out, { recursive: true });await page.emulateMedia({reducedMotion:"reduce"});await snap("after-reduced-motion");assert.deepEqual(errors,[]);- console.log("PASS: generated questions, typing, voice affordance, explicit save, failed-save retry, reload, compact playback, mobile, reduced motion, synthetic Live session, multi-field fill, undo, microphone cleanup, no browser errors.");+ console.log("PASS: generated questions, typing, voice affordance, explicit save, save during recording and pending fill, failed-save retry, reload, compact playback, mobile, reduced motion, synthetic Live session, multi-field fill, undo, microphone cleanup, no browser errors.");} finally { await browser.close(); }})().catch(e=>{console.error(e);process.exitCode=1;});diff --git a/packages/workflows-ui/src/workflow-questions.tsx b/packages/workflows-ui/src/workflow-questions.tsxindex 074771f89..9456119a3 100644--- a/packages/workflows-ui/src/workflow-questions.tsx+++ b/packages/workflows-ui/src/workflow-questions.tsx@@ -43,6 +43,8 @@ export function WorkflowQuestions({ workflow, save, voice, active = true }: {const saving = useRef(false);const session = useRef<QuestionnaireVoiceSession>();const filler = useRef<LiveVoiceFill>();+ const finishVoice = useRef<() => void>();+ const [voiceSession, setVoiceSession] = useState(0);const prefix = useRef("");const id = useId();const recording = voiceState.status === "connecting" || voiceState.status === "listening";@@ -80,8 +82,21 @@ export function WorkflowQuestions({ workflow, save, voice, active = true }: {transcriptRef.current = combined; setTranscript(combined); fill.enqueue(combined);});session.current = connection;- return () => { mounted = false; connection.stop(); fill.dispose(); abort.abort(); session.current = undefined; filler.current = undefined; setVoiceState(s => ({ ...s, status: "stopped", level: undefined })); };- }, [voice, active]);+ const dispose = () => {+ mounted = false;+ // Invalidate callbacks before stopping: Stop normally queues a final fill.+ connection.stop(); fill.dispose(); abort.abort();+ session.current = undefined; filler.current = undefined; finishVoice.current = undefined;+ setFilling(false);+ setVoiceState(s => ({ ...s, status: "stopped", level: undefined }));+ };+ finishVoice.current = () => {+ dispose();+ // A later recording gets a fresh controller and cannot revive stale fills.+ setVoiceSession(value => value + 1);+ };+ return dispose;+ }, [voice, active, voiceSession]);function manual(question: string, value: string) {locked.current.add(question);setAnswers(a => ({ ...a, [question]: value })); setNotice("");@@ -93,7 +108,8 @@ export function WorkflowQuestions({ workflow, save, voice, active = true }: {void session.current?.start(workflow.title.slice(0, 300), questions.slice(0, 30));}async function submit() {- if (!save || !changed || recording || filling || saving.current) return;+ if (!save || !changed || saving.current) return;+ if (recording || filling) finishVoice.current?.();saving.current = true; setBusy(true); setError(""); setNotice("");try {const next = await save(writeWorkflowAnswers(workflow.userCorrection, fields.filter(q => (answers[q] ?? "").trim() !== (saved[q] ?? "").trim()).map(question => ({ question, answer: answers[question] ?? "" }))));@@ -136,7 +152,7 @@ export function WorkflowQuestions({ workflow, save, voice, active = true }: {</div></div>)}{error && <p className={styles.error} role="alert">{error}</p>}- {save && <div className={styles.actions}><span role="status">{notice || "Review before saving. Saved answers refine future workflow updates."}</span><button type="submit" disabled={!changed || busy || recording || filling}>{busy ? "Saving…" : "Save answers"}</button></div>}+ {save && <div className={styles.actions}><span role="status">{notice || (recording || filling ? "Saving stops voice and keeps the answers shown here." : "Saved answers refine future workflow updates.")}</span><button type="submit" disabled={!changed || busy}>{busy ? "Saving…" : "Save answers"}</button></div>}</form></section>;}