Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(workflows): disclose captured sources and preview step media

louisaf501c9a2c5b
diff --git a/apps/screenpipe-app-tauri/components/workflows/step-evidence.test.tsx b/apps/screenpipe-app-tauri/components/workflows/step-evidence.test.tsx
new file mode 100644
index 000000000..bc8b4315e
--- /dev/null
+++ b/apps/screenpipe-app-tauri/components/workflows/step-evidence.test.tsx
@@ -0,0 +1,60 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import React from "react";
+import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { WorkflowStepEvidence } from "../../../../packages/workflows-ui/src/workflow-step-evidence";
+import { fixtureWorkflowAnalysis } from "../../../../packages/workflows-ui/src/fixture-platform";
+import type { WorkflowsPlatform } from "../../../../packages/workflows-ui/src/platform";
+
+const workflow = fixtureWorkflowAnalysis.analysis.workflows[0];
+const stage = workflow.stages[0];
+const close = () => fireEvent.click(screen.getByRole("button", { name: `Close recording for ${stage.name}` }));
+const play = () => fireEvent.click(screen.getByRole("button", { name: `View recording for ${stage.name}` }));
+afterEach(cleanup);
+
+describe("step evidence media", () => {
+ it("loads only on request and releases the media when closed", async () => {
+ const load = vi.fn().mockResolvedValue({ kind: "video", url: "https://fixture.invalid/clip.mp4", timestamp: stage.evidence[0].timestamp, frameId: 1, offsetSeconds: 2 });
+ const release = vi.fn().mockResolvedValue(undefined);
+ render(<WorkflowStepEvidence workflow={workflow} stage={stage} platform={{ loadWorkflowRecording: load, releaseWorkflowRecording: release } as unknown as WorkflowsPlatform} />);
+ expect(load).not.toHaveBeenCalled();
+ play();
+ const video = await screen.findByLabelText("Local recording");
+ expect(video).toHaveAttribute("controls");
+ Object.defineProperty(video, "duration", { value: 30 });
+ fireEvent.loadedMetadata(video);
+ expect((video as HTMLVideoElement).currentTime).toBe(2);
+ fireEvent.loadedData(video);
+ close();
+ await waitFor(() => expect(release).toHaveBeenCalledWith("https://fixture.invalid/clip.mp4"));
+ expect(screen.queryByLabelText("Local recording")).toBeNull();
+ });
+
+ it("shows a failed load and retries on request", async () => {
+ const load = vi.fn().mockRejectedValueOnce(new Error("offline")).mockResolvedValueOnce(null);
+ render(<WorkflowStepEvidence workflow={workflow} stage={stage} platform={{ loadWorkflowRecording: load } as unknown as WorkflowsPlatform} />);
+ play();
+ expect(await screen.findByRole("alert")).toHaveTextContent("Recording unavailable");
+ fireEvent.click(screen.getByRole("button", { name: "Retry" }));
+ await waitFor(() => expect(load).toHaveBeenCalledTimes(2));
+ await waitFor(() => expect(screen.queryByRole("alert")).toBeNull());
+ });
+
+ it("does not fabricate screenshots or offer playback without a source", () => {
+ render(<WorkflowStepEvidence workflow={workflow} stage={{ ...stage, screenshot: null, evidence: [], procedure: [] }} platform={{ loadWorkflowRecording: vi.fn() } as unknown as WorkflowsPlatform} />);
+ expect(screen.queryByRole("img")).toBeNull();
+ expect(screen.queryByRole("button", { name: /View recording/ })).toBeNull();
+ });
+
+ it("ignores a response arriving after the player is closed", async () => {
+ let resolve!: (value: unknown) => void;
+ const load = vi.fn().mockReturnValue(new Promise(done => { resolve = done; }));
+ const release = vi.fn().mockResolvedValue(undefined);
+ render(<WorkflowStepEvidence workflow={workflow} stage={stage} platform={{ loadWorkflowRecording: load, releaseWorkflowRecording: release } as unknown as WorkflowsPlatform} />);
+ play(); close();
+ resolve({ kind: "video", url: "https://fixture.invalid/late.mp4" });
+ await waitFor(() => expect(release).toHaveBeenCalledWith("https://fixture.invalid/late.mp4"));
+ expect(screen.queryByLabelText("Local recording")).toBeNull();
+ });
+});
diff --git a/apps/screenpipe-workflows-web/scripts/eval-workflow-editor.cjs b/apps/screenpipe-workflows-web/scripts/eval-workflow-editor.cjs
index 4e0282af1..b60c1cb80 100644
--- a/apps/screenpipe-workflows-web/scripts/eval-workflow-editor.cjs
+++ b/apps/screenpipe-workflows-web/scripts/eval-workflow-editor.cjs
@@ -44,13 +44,27 @@ require("node:fs").mkdirSync(out, { recursive: true });
await page.waitForTimeout(150);
assert.equal(await button("Move step 2").evaluate(e => getComputedStyle(e).opacity), "1");
await field("Workflow title").focus();
- await page.getByText("Sources and screenshots", { exact: true }).first().click();
- await page.locator("p").filter({ hasText: "Collect sources was observed in this fictional workspace." }).waitFor();
- await page.getByText("Sources and screenshots", { exact: true }).first().click();
+ const references = page.getByRole("region", { name: "References for Collect sources", exact: true });
+ await references.getByText("1 source", { exact: true }).click();
+ const raw = references.getByRole("region", { name: "Captured text 1 for Collect sources", exact: true });
+ assert.equal(await raw.isVisible(), false);
+ await references.getByText("Captured text", { exact: true }).click();
+ await raw.waitFor();
+ await references.getByText("1 source", { exact: true }).click();
+ const enlarge = references.getByRole("button", { name: "Enlarge screenshot for Collect sources" });
+ const smallWidth = (await enlarge.boundingBox()).width;
+ await enlarge.click();
+ const reduce = references.getByRole("button", { name: "Reduce screenshot for Collect sources" });
+ assert((await reduce.boundingBox()).width > smallWidth);
+ await page.screenshot({ path: out + "/screenshot-expanded.png" });
+ await reduce.click();
+ await references.getByRole("button", { name: "View recording for Collect sources" }).click();
+ await references.getByRole("img", { name: "Captured moment for Collect sources", exact: true }).waitFor();
+ await references.getByRole("button", { name: "Close recording for Collect sources" }).click();
});
await check("drag step, auto-save and retain its screenshot", async () => {
await page.getByRole("article", { name: "Step 1", exact: true }).hover();
- await button("Move step 1").dragTo(page.getByRole("article", { name: "Step 2", exact: true }));
+ await button("Move step 1").dragTo(field("Step 2 title"));
assert.equal(await field("Step 2 title").inputValue(), "Collect sources");
await saved();
const workflow = await stored();
@@ -74,9 +88,13 @@ require("node:fs").mkdirSync(out, { recursive: true });
const after = await stored();
assert.equal(after.stages[2].name, "Collect sources");
assert.deepEqual(after.stages[2].screenshot, before.stages[0].screenshot);
- await page.getByRole("article", { name: "Step 3", exact: true }).hover();
- await button("Move step 3").dragTo(page.getByRole("article", { name: "Step 1", exact: true }));
+ await button("Move step 3").focus();
+ await page.keyboard.press("Alt+ArrowUp");
await saved();
+ await button("Move step 2").focus();
+ await page.keyboard.press("Alt+ArrowUp");
+ await saved();
+ assert.equal((await stored()).stages[0].name, "Collect sources");
});
await check("typing auto-saves without losing focus", async () => {
await field("Workflow title").fill("Research brief");
@@ -179,14 +197,25 @@ require("node:fs").mkdirSync(out, { recursive: true });
const stage = catalog.analysis.workflows[0].stages[0];
stage.procedure = [];
stage.screenshot = null;
+ stage.evidence[0].detail = "Fictional capture: navigation toolbar sidebar unread messages. ".repeat(700);
const context = await browser.newContext();
await context.addInitScript(data => localStorage.setItem("screenpipe:fictional-workflow-editor-preview", JSON.stringify(data)), catalog);
+ context.setDefaultTimeout(10000);
const legacy = await context.newPage();
await legacy.goto(process.env.WORKFLOWS_PREVIEW_URL || "http://127.0.0.1:1431/preview");
await legacy.getByRole("button", { name: "Open map", exact: true }).first().click();
- await legacy.getByText("Sources and screenshots", { exact: true }).first().click();
- await legacy.locator("p").filter({ hasText: stage.evidence[0].detail }).waitFor();
- assert.equal(await legacy.getByRole("article", { name: "Step 1", exact: true }).getByText("No captured source attached.").count(), 0);
+ const references = legacy.getByRole("region", { name: `References for ${stage.name}`, exact: true });
+ await references.getByText("1 source", { exact: true }).click();
+ const raw = references.getByRole("region", { name: `Captured text 1 for ${stage.name}`, exact: true });
+ assert.equal(await raw.isVisible(), false);
+ await references.getByText("Captured text", { exact: true }).click();
+ assert.equal(await raw.textContent(), stage.evidence[0].detail);
+ assert((await raw.boundingBox()).height <= 185);
+ assert(await raw.evaluate(e => e.scrollHeight > e.clientHeight));
+ await legacy.screenshot({ path: out + "/long-source.png" });
+ await legacy.setViewportSize({ width: 390, height: 900 });
+ await legacy.getByRole("button", { name: "Collapse left sidebar", exact: true }).click();
+ assert(await raw.evaluate(e => e.getBoundingClientRect().right <= innerWidth));
await context.close();
});
await check("touch layout keeps controls hidden until a step is focused", async () => {
diff --git a/packages/workflows-ui/src/fixture-platform.ts b/packages/workflows-ui/src/fixture-platform.ts
index 19c4dc8ee..8d3993f95 100644
--- a/packages/workflows-ui/src/fixture-platform.ts
+++ b/packages/workflows-ui/src/fixture-platform.ts
@@ -130,7 +130,7 @@ function fixtureWorkflow(input: FixtureWorkflow, index: number): WorkflowMap {
observedOccurrences: 5 + index,
observedDays: 4 + index,
evidence: [allEvidence[stageIndex]],
- screenshot: screenshot(name, input.apps[Math.min(stageIndex, input.apps.length - 1)], index * 10 + stageIndex + 1),
+ screenshot: { ...screenshot(name, input.apps[Math.min(stageIndex, input.apps.length - 1)], index * 10 + stageIndex + 1), timestamp: allEvidence[stageIndex].timestamp, matchDistanceSeconds: 0 },
})),
bottlenecks: [{
label: input.friction,
@@ -384,7 +384,7 @@ function fixtureSkillReceipt(draft: WorkflowSkillDraft) {
};
}
-async function rasterizeFixture(workflow: WorkflowMap) {
+async function rasterizeFixture(workflow: Pick<WorkflowMap, "stages">) {
// Rasterize this file's fictional SVGs so previews exercise the same raster-only export.
for (const stage of workflow.stages) {
if (!stage.screenshot?.dataUrl.startsWith("data:image/svg+xml")) continue;
@@ -448,8 +448,14 @@ export function createFixtureWorkflowsPlatform(analysis: WorkflowAnalysis = fixt
onField({ field: "company", value: "Preview company: a team that helps businesses resolve customer requests." });
onField({ field: "summary", value: "Preview role: I manage customer operations and improve how our team handles requests." });
},
+ loadWorkflowRecording: async (timestamp, app) => {
+ const stage = current.analysis.workflows.flatMap(w => w.stages).find(s => s.evidence.some(e => e.timestamp === timestamp && e.app === app));
+ if (!stage?.screenshot) return null;
+ await rasterizeFixture({ stages: [stage] });
+ return { kind: "image", url: stage.screenshot.dataUrl, frameId: stage.screenshot.frameId, timestamp: stage.screenshot.timestamp, offsetSeconds: 0, matchDistanceSeconds: 0 };
+ },
ensureRuntime: async () => fixtureWorkflowRuntime,
- loadCapturedWork: async () => restore(),
+ loadCapturedWork: async () => { const value = restore(); await Promise.all(value.analysis.workflows.map(rasterizeFixture)); return structuredClone(value); },
analyzeCapturedWork: async () => restore(),
saveWorkflowEdits: async (draft) => {
restore();
diff --git a/packages/workflows-ui/src/workflow-editor.module.css b/packages/workflows-ui/src/workflow-editor.module.css
index 47022c1fe..454f13d0e 100644
--- a/packages/workflows-ui/src/workflow-editor.module.css
+++ b/packages/workflows-ui/src/workflow-editor.module.css
@@ -39,7 +39,7 @@
.stepsHeading { margin: 18px 0 8px; padding: 0 5px; }
.stepsHeading strong { color: var(--muted); font-size: 11px; font-weight: 500; }
.steps { margin: 0 0 4px; padding-left: 26px; }
-.step { position: relative; padding: 4px 0; margin: 0 0 12px; border: 0; background: transparent; }
+.step { position: relative; padding: 4px 0; margin: 0 0 24px; border: 0; background: transparent; }
.step.drop { box-shadow: 0 -2px 0 var(--ink); }
.stepTop { padding-right: 30px; position: relative; display: grid; grid-template-columns: 24px minmax(0, 1fr) auto; align-items: start; }
.number { padding-top: 7px; color: var(--muted); font-size: 12px; font-variant-numeric: tabular-nums; }
diff --git a/packages/workflows-ui/src/workflow-step-evidence.module.css b/packages/workflows-ui/src/workflow-step-evidence.module.css
new file mode 100644
index 000000000..ef21d3c5a
--- /dev/null
+++ b/packages/workflows-ui/src/workflow-step-evidence.module.css
@@ -0,0 +1,29 @@
+/* screenpipe — AI that knows everything you've seen, said, or heard */
+/* https://screenpipe.com */
+.evidence { margin: 12px 0 4px; min-width: 0; }
+.figure { max-width: 360px; margin: 0 0 6px; }
+.figure.expanded { max-width: 100%; }
+.figure .imageButton { display: block; padding: 0; position: relative; width: 100%; border: 1px solid var(--line); border-radius: 6px; overflow: hidden; background: var(--soft); cursor: zoom-in; }
+.expanded .imageButton { cursor: zoom-out; }
+.imageButton img { display: block; width: 100%; max-height: 520px; object-fit: contain; }
+.zoom { position: absolute; right: 8px; bottom: 8px; display: grid; place-items: center; padding: 6px; background: var(--panel); border: 1px solid var(--line); border-radius: 4px; opacity: 0; }
+.imageButton:hover .zoom, .imageButton:focus-visible .zoom { opacity: 1; }
+.figure figcaption { display: flex; justify-content: space-between; align-items: center; gap: 8px; font-size: 11px; line-height: 1.5; color: var(--muted); }
+.figure figcaption > span { min-width: 0; overflow-wrap: anywhere; }
+.links { display: flex; flex-wrap: wrap; align-items: flex-start; gap: 4px 12px; font-size: 11px; color: var(--muted); }
+.links > button { font-size: 11px; }
+.sources { flex: 1; min-width: min(240px, 100%); }
+.evidence summary { display: flex; align-items: center; gap: 6px; width: fit-content; min-height: 28px; list-style: none; cursor: pointer; font-size: 11px; color: var(--muted); }
+.evidence summary::-webkit-details-marker { display: none; }
+.evidence summary:focus-visible { outline: 2px solid var(--ink); outline-offset: 2px; }
+.evidence details[open] > summary > svg { transform: rotate(180deg); }
+.sourceList { max-height: 320px; overflow: auto; overscroll-behavior: contain; border-left: 1px solid var(--line); margin: 4px 0 8px; padding-left: 12px; }
+.rawSource > summary { width: 100%; justify-content: space-between; flex-wrap: wrap; padding: 4px 0; }
+.rawSource > summary > span:last-child { display: inline-flex; gap: 4px; align-items: center; }
+.rawText { max-height: 160px; overflow: auto; overscroll-behavior: contain; white-space: pre-wrap; overflow-wrap: anywhere; font-size: 12px; line-height: 1.6; padding: 10px 12px; background: var(--soft); border-radius: 4px; color: var(--muted); }
+.questions { font-size: 12px; color: var(--muted); }
+.questions ul { padding-left: 18px; line-height: 1.6; }
+.player { position: relative; padding-top: 22px; }
+.player > section { margin: 0 0 8px; border-radius: 6px; }
+.player .close { position: absolute; right: 0; top: -6px; }
+@media (max-width: 560px) { .figure { max-width: 100%; } }
diff --git a/packages/workflows-ui/src/workflow-step-evidence.tsx b/packages/workflows-ui/src/workflow-step-evidence.tsx
new file mode 100644
index 000000000..4d092becf
--- /dev/null
+++ b/packages/workflows-ui/src/workflow-step-evidence.tsx
@@ -0,0 +1,52 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+"use client";
+
+import { useMemo, useState } from "react";
+import { ChevronDown, Maximize2, Minimize2, Play, X } from "lucide-react";
+import type { WorkflowMap, WorkflowStage } from "./model";
+import type { WorkflowsPlatform } from "./platform";
+import { CapturedMomentButton, WorkflowReplay } from "./workflow-replay";
+import styles from "./workflow-step-evidence.module.css";
+
+const when = (value: string) => Number.isFinite(Date.parse(value))
+ ? new Date(value).toLocaleString(undefined, { month: "short", day: "numeric", hour: "numeric", minute: "2-digit" }) : "Date unavailable";
+
+export function WorkflowStepEvidence({ workflow, stage, platform }: {
+ workflow: WorkflowMap; stage: WorkflowStage; platform: WorkflowsPlatform;
+}) {
+ const [expanded, setExpanded] = useState(false);
+ const [playing, setPlaying] = useState(false);
+ const replay = useMemo(() => ({ ...workflow, stages: [stage] }), [workflow, stage]);
+ const screenshot = stage.screenshot?.visualVerified ? stage.screenshot : null;
+ const sources = [...(stage.procedure ?? []).filter(p => p.quote).map(p => ({ text: p.quote, timestamp: p.timestamp, app: p.app })),
+ ...stage.evidence.map(e => ({ text: e.detail, timestamp: e.timestamp, app: e.app }))]
+ .filter((entry, index, all) => entry.text && all.findIndex(e => e.text === entry.text && e.timestamp === entry.timestamp && e.app === entry.app) === index);
+ const canReplay = platform.loadWorkflowRecording && stage.evidence.some(e => !["audio", "meeting"].includes(e.source ?? "") && Number.isFinite(Date.parse(e.timestamp)));
+ return <section className={`${styles.evidence} ph-no-capture ph-mask`} aria-label={`References for ${stage.name}`}>
+ {screenshot && !playing && <figure className={`${styles.figure} ${expanded ? styles.expanded : ""}`}>
+ <button className={styles.imageButton} type="button" aria-label={`${expanded ? "Reduce" : "Enlarge"} screenshot for ${stage.name}`} aria-expanded={expanded} onClick={() => setExpanded(!expanded)}>
+ <img src={screenshot.dataUrl} alt={`Captured reference for ${stage.name}`} loading="lazy" draggable={false} data-lm-disable="true" />
+ <span className={styles.zoom}>{expanded ? <Minimize2 size={15} /> : <Maximize2 size={15} />}</span>
+ </button>
+ <figcaption><span>{screenshot.app} · {when(screenshot.timestamp)}</span><CapturedMomentButton compact frameId={screenshot.frameId} timestamp={screenshot.timestamp} open={platform.openCapturedMoment} /></figcaption>
+ </figure>}
+ {playing && <div className={styles.player}>
+ <button type="button" className={styles.close} aria-label={`Close recording for ${stage.name}`} title="Close recording" onClick={() => setPlaying(false)}><X size={15} /></button>
+ <WorkflowReplay workflow={replay} loadRecording={platform.loadWorkflowRecording} releaseRecording={platform.releaseWorkflowRecording} openCapturedMoment={platform.openCapturedMoment} />
+ </div>}
+ <div className={styles.links}>
+ {canReplay && !playing && <button type="button" onClick={() => setPlaying(true)} aria-label={`View recording for ${stage.name}`}><Play size={13} />View recording</button>}
+ {!!sources.length && <details className={styles.sources}>
+ <summary>{sources.length} source{sources.length === 1 ? "" : "s"}<ChevronDown size={12} /></summary>
+ <div className={styles.sourceList}>
+ {sources.map((source, index) => <details className={styles.rawSource} key={`${source.timestamp}-${index}`}>
+ <summary><span>{source.app || "Captured source"} · {when(source.timestamp)}</span><span>Captured text<ChevronDown size={12} /></span></summary>
+ <div className={styles.rawText} role="region" aria-label={`Captured text ${index + 1} for ${stage.name}`} tabIndex={0}>{source.text}</div>
+ </details>)}
+ </div>
+ </details>}
+ </div>
+ {!!stage.openQuestions?.length && <details className={styles.questions}><summary>Unresolved details<ChevronDown size={12} /></summary><ul>{stage.openQuestions.map(q => <li key={q}>{q}</li>)}</ul></details>}
+ </section>;
+}
diff --git a/packages/workflows-ui/src/workflows-app.tsx b/packages/workflows-ui/src/workflows-app.tsx
index 7d10a37da..5aa84c530 100644
--- a/packages/workflows-ui/src/workflows-app.tsx
+++ b/packages/workflows-ui/src/workflows-app.tsx
@@ -4,6 +4,7 @@
"use client";
import { useNavigationWidth } from "./use-navigation-width";
+import { WorkflowStepEvidence } from "./workflow-step-evidence";
import { WorkflowEditor } from "./workflow-editor";
import { retainNewerWorkflowEdits, type WorkflowEdit } from "./workflow-edits";
import { WorkflowModelControl } from "./model-choice";
@@ -947,15 +948,7 @@ function WorkflowDetail({ workflow, navigate, platform, workProfile, saveCorrect
return (
<>
<button className={styles.backButton} onClick={() => navigate("workflows")}><ArrowLeft size={14} />All workflows</button>
- {saveEdits ? <WorkflowEditor key={workflow.id || workflow.title} workflow={workflow} save={saveEdits} actions={workflowActions} renderSource={stage => <details className={styles.workflowQuality}>
- <summary><div><strong>Sources and screenshots</strong></div><ChevronDown size={14} /></summary>
- <p>{stage.apps.join(" · ")} · {stage.observedOccurrences} observations across {stage.observedDays} days</p>
- {stage.screenshot?.visualVerified && <div className={styles.stageScreenshot}><div className={styles.screenshotFrame}><img src={stage.screenshot.dataUrl} alt={`Captured reference for ${stage.name}`} draggable={false} data-lm-disable="true" /></div><CapturedMomentButton frameId={stage.screenshot.frameId} timestamp={stage.screenshot.timestamp} open={platform.openCapturedMoment} /></div>}
- {stage.procedure?.filter(p => p.quote).map((p, i) => <blockquote key={i}>{p.quote}<small>{p.app} · {formatEvidenceTimestamp(p.timestamp)}</small></blockquote>)}
- {stage.evidence.filter(e => !stage.procedure?.some(p => p.quote === e.detail && p.timestamp === e.timestamp)).map((e, i) => <p key={`evidence-${i}`}>{e.detail}<small>{e.app} · {formatEvidenceTimestamp(e.timestamp)}</small></p>)}
- {stage.openQuestions?.map((question, i) => <p key={i}>{question}</p>)}
- {!stage.screenshot?.visualVerified && !stage.procedure?.some(p => p.quote) && !stage.evidence.length && <p>No captured source attached.</p>}
- </details>} /> : <section className={styles.detailHeader}>
+ {saveEdits ? <WorkflowEditor key={workflow.id || workflow.title} workflow={workflow} save={saveEdits} actions={workflowActions} renderSource={stage => <WorkflowStepEvidence workflow={workflow} stage={stage} platform={platform} />} /> : <section className={styles.detailHeader}>
<div><Pill>Evidence on {workflow.repetitions} captured day{workflow.repetitions === 1 ? "" : "s"}</Pill><h1>{workflow.title}</h1><p>{workflow.description}</p>{workflowActions}</div>
{timing ? <div className={styles.detailTotal}><span>{timing.sampleCount > 1 ? "Avg. time / run" : "Time for one run"}</span><strong>{formatEstimatedMinutes(timing.averageMinutes)}</strong><small>{timing.sampleCount} run{timing.sampleCount === 1 ? "" : "s"} · estimated elapsed time</small></div> : measuredDuration && <div className={styles.detailTotal}><span>Observed meeting duration</span><strong>{workflowDurationLabel(workflow)}</strong></div>}
</section>}