Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(workflows): allow saving answers during voice processing (#7436)

Louis Beaumontdbde4dcb8b72
diff --git a/apps/screenpipe-app-tauri/components/workflows/workflow-questions.test.tsx b/apps/screenpipe-app-tauri/components/workflows/workflow-questions.test.tsx
index 776f7f98c..03520b0bb 100644
--- a/apps/screenpipe-app-tauri/components/workflows/workflow-questions.test.tsx
+++ b/apps/screenpipe-app-tauri/components/workflows/workflow-questions.test.tsx
@@ -1,7 +1,7 @@
// screenpipe — AI that knows everything you've seen, said, or heard
// https://screenpipe.com
import React from "react";
-import { fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
import { describe, expect, it, vi } from "vitest";
import { WorkflowQuestions } from "../../../../packages/workflows-ui/src/workflow-questions";
import { readWorkflowAnswers, writeWorkflowAnswers } from "../../../../packages/workflows-ui/src/workflow-answers";
@@ -82,7 +82,7 @@ it("uses a section voice control, protects manual edits during inference, suppor
fireEvent.click(screen.getByRole("button", { name: "Stop recording" }));
await waitFor(() => expect(voice.fill).toHaveBeenCalled());
fireEvent.change(screen.getByLabelText(question), { target: { value: "Typed while filling" } });
- expect(screen.getByRole("button", { name: "Save answers" })).toBeDisabled();
+ expect(screen.getByRole("button", { name: "Save answers" })).toBeEnabled();
resolve([{ question, answer: "The lead", quote: "The lead reviews it." }, { question: "Where is it saved?", answer: "Shared drive", quote: "shared drive" }]);
await waitFor(() => expect(screen.getByLabelText("Where is it saved?")).toHaveValue("Shared drive"));
expect(screen.getByLabelText(question)).toHaveValue("Typed while filling");
@@ -117,3 +117,37 @@ it("does not remount step media when saving answers only changes the revision",
expect(screen.getAllByTestId("step-media")[0]).toBe(original);
expect(save).not.toHaveBeenCalled();
});
+
+
+it.each(["recording", "filling"])("saves typed answers during %s and ignores late voice results", async phase => {
+ let finish!: (value: any) => void;
+ let fillSignal: AbortSignal | undefined;
+ const voice: QuestionnaireVoice = { connect: vi.fn(), disconnect: vi.fn(), fill: vi.fn((_input, signal) => {
+ fillSignal = signal;
+ return new Promise(resolve => { finish = resolve; });
+ }) };
+ const save = vi.fn(async (userCorrection: string) => ({ ...workflow, userCorrection }));
+ render(<WorkflowQuestions workflow={workflow} save={save} voice={voice} />);
+ fireEvent.click(screen.getByRole("button", { name: "Answer with voice" }));
+ act(() => live.transcript?.("The lead reviews it. Shared drive."));
+ if (phase === "filling") {
+ fireEvent.click(screen.getByRole("button", { name: "Stop recording" }));
+ await waitFor(() => expect(voice.fill).toHaveBeenCalled());
+ }
+ fireEvent.change(screen.getByLabelText(question), { target: { value: "My reviewed answer" } });
+ const button = screen.getByRole("button", { name: "Save answers" });
+ expect(button).toBeEnabled();
+ fireEvent.click(button);
+ await screen.findByText("Answers saved for the next workflow update.");
+ expect(save).toHaveBeenCalledTimes(1);
+ expect(readWorkflowAnswers(save.mock.calls[0][0])).toEqual([{ question, answer: "My reviewed answer" }]);
+ expect(screen.queryByRole("button", { name: "Stop recording" })).not.toBeInTheDocument();
+ if (phase === "filling") {
+ expect(fillSignal?.aborted).toBe(true);
+ await act(async () => finish([{ question: "Where is it saved?", answer: "Late answer", quote: "Shared drive." }]));
+ expect(screen.getByLabelText("Where is it saved?")).toHaveValue("");
+ }
+ expect(button).toBeDisabled();
+ fireEvent.click(screen.getByRole("button", { name: "Answer with voice" }));
+ expect(screen.getByRole("button", { name: "Stop recording" })).toBeInTheDocument();
+});
diff --git a/apps/screenpipe-workflows-web/scripts/eval-workflow-questions.cjs b/apps/screenpipe-workflows-web/scripts/eval-workflow-questions.cjs
index d39855666..a5d77b53a 100644
--- a/apps/screenpipe-workflows-web/scripts/eval-workflow-questions.cjs
+++ b/apps/screenpipe-workflows-web/scripts/eval-workflow-questions.cjs
@@ -109,6 +109,37 @@ fs.mkdirSync(out, { recursive: true });
await snap("after-saved");
await open();
assert.equal(await input.inputValue(), "The project lead reviews the brief before it is shared with the team.");
+ // Saving manual answers must not wait for a recording or an AI fill.
+ for (const phase of ["recording", "filling"]) {
+ releaseFill = undefined;
+ await questions.getByRole("button", { name: "Answer with voice" }).click();
+ await page.getByText("Listening", { exact: true }).waitFor();
+ await page.evaluate(() => window.__voiceChannel.onmessage({ data: JSON.stringify({
+ type: "session.input_transcript.delta", event_id: "save-test", delta: "The project lead reviews it.",
+ }) }));
+ if (phase === "filling") {
+ await page.getByRole("button", { name: "Stop recording" }).click();
+ const deadline = Date.now() + 10000;
+ while (!releaseFill && Date.now() < deadline) await page.waitForTimeout(50);
+ assert.ok(releaseFill, "Fill request should be in flight");
+ }
+ const answer = `Reviewed answer during ${phase}`;
+ await input.fill(answer);
+ const saveButton = page.getByRole("button", { name: "Save answers", exact: true });
+ assert.equal(await saveButton.isEnabled(), true);
+ await saveButton.scrollIntoViewIfNeeded();
+ await snap(`save-during-${phase}`);
+ await saveButton.click();
+ await page.getByText("Answers saved for the next workflow update.", { exact: true }).waitFor();
+ assert.equal(await page.getByRole("button", { name: "Stop recording" }).count(), 0);
+ if (releaseFill) releaseFill();
+ await open();
+ assert.equal(await input.inputValue(), answer);
+ }
+ // Restore the answer expected by the remaining layout checks.
+ await input.fill("The project lead reviews the brief before it is shared with the team.");
+ await page.getByRole("button", { name: "Save answers", exact: true }).click();
+ await page.getByText("Answers saved for the next workflow update.", { exact: true }).waitFor();
const play=page.getByRole("button",{name:/View recording for/}).first();
await play.scrollIntoViewIfNeeded();
assert.equal((await play.innerText()).trim(), "");
@@ -131,6 +162,6 @@ fs.mkdirSync(out, { recursive: true });
await page.emulateMedia({reducedMotion:"reduce"});
await snap("after-reduced-motion");
assert.deepEqual(errors,[]);
- console.log("PASS: generated questions, typing, voice affordance, explicit save, failed-save retry, reload, compact playback, mobile, reduced motion, synthetic Live session, multi-field fill, undo, microphone cleanup, no browser errors.");
+ console.log("PASS: generated questions, typing, voice affordance, explicit save, save during recording and pending fill, failed-save retry, reload, compact playback, mobile, reduced motion, synthetic Live session, multi-field fill, undo, microphone cleanup, no browser errors.");
} finally { await browser.close(); }
})().catch(e=>{console.error(e);process.exitCode=1;});
diff --git a/packages/workflows-ui/src/workflow-questions.tsx b/packages/workflows-ui/src/workflow-questions.tsx
index 074771f89..9456119a3 100644
--- a/packages/workflows-ui/src/workflow-questions.tsx
+++ b/packages/workflows-ui/src/workflow-questions.tsx
@@ -43,6 +43,8 @@ export function WorkflowQuestions({ workflow, save, voice, active = true }: {
const saving = useRef(false);
const session = useRef<QuestionnaireVoiceSession>();
const filler = useRef<LiveVoiceFill>();
+ const finishVoice = useRef<() => void>();
+ const [voiceSession, setVoiceSession] = useState(0);
const prefix = useRef("");
const id = useId();
const recording = voiceState.status === "connecting" || voiceState.status === "listening";
@@ -80,8 +82,21 @@ export function WorkflowQuestions({ workflow, save, voice, active = true }: {
transcriptRef.current = combined; setTranscript(combined); fill.enqueue(combined);
});
session.current = connection;
- return () => { mounted = false; connection.stop(); fill.dispose(); abort.abort(); session.current = undefined; filler.current = undefined; setVoiceState(s => ({ ...s, status: "stopped", level: undefined })); };
- }, [voice, active]);
+ const dispose = () => {
+ mounted = false;
+ // Invalidate callbacks before stopping: Stop normally queues a final fill.
+ connection.stop(); fill.dispose(); abort.abort();
+ session.current = undefined; filler.current = undefined; finishVoice.current = undefined;
+ setFilling(false);
+ setVoiceState(s => ({ ...s, status: "stopped", level: undefined }));
+ };
+ finishVoice.current = () => {
+ dispose();
+ // A later recording gets a fresh controller and cannot revive stale fills.
+ setVoiceSession(value => value + 1);
+ };
+ return dispose;
+ }, [voice, active, voiceSession]);
function manual(question: string, value: string) {
locked.current.add(question);
setAnswers(a => ({ ...a, [question]: value })); setNotice("");
@@ -93,7 +108,8 @@ export function WorkflowQuestions({ workflow, save, voice, active = true }: {
void session.current?.start(workflow.title.slice(0, 300), questions.slice(0, 30));
}
async function submit() {
- if (!save || !changed || recording || filling || saving.current) return;
+ if (!save || !changed || saving.current) return;
+ if (recording || filling) finishVoice.current?.();
saving.current = true; setBusy(true); setError(""); setNotice("");
try {
const next = await save(writeWorkflowAnswers(workflow.userCorrection, fields.filter(q => (answers[q] ?? "").trim() !== (saved[q] ?? "").trim()).map(question => ({ question, answer: answers[question] ?? "" }))));
@@ -136,7 +152,7 @@ export function WorkflowQuestions({ workflow, save, voice, active = true }: {
</div>
</div>)}
{error && <p className={styles.error} role="alert">{error}</p>}
- {save && <div className={styles.actions}><span role="status">{notice || "Review before saving. Saved answers refine future workflow updates."}</span><button type="submit" disabled={!changed || busy || recording || filling}>{busy ? "Saving…" : "Save answers"}</button></div>}
+ {save && <div className={styles.actions}><span role="status">{notice || (recording || filling ? "Saving stops voice and keeps the answers shown here." : "Saved answers refine future workflow updates.")}</span><button type="submit" disabled={!changed || busy}>{busy ? "Saving…" : "Save answers"}</button></div>}
</form>
</section>;
}