YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(workflows): disclose captured sources and preview step media
louisaf501c9a2c5b
diff --git a/apps/screenpipe-app-tauri/components/workflows/step-evidence.test.tsx b/apps/screenpipe-app-tauri/components/workflows/step-evidence.test.tsxnew file mode 100644index 000000000..bc8b4315e--- /dev/null+++ b/apps/screenpipe-app-tauri/components/workflows/step-evidence.test.tsx@@ -0,0 +1,60 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import React from "react";+import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";+import { afterEach, describe, expect, it, vi } from "vitest";+import { WorkflowStepEvidence } from "../../../../packages/workflows-ui/src/workflow-step-evidence";+import { fixtureWorkflowAnalysis } from "../../../../packages/workflows-ui/src/fixture-platform";+import type { WorkflowsPlatform } from "../../../../packages/workflows-ui/src/platform";++const workflow = fixtureWorkflowAnalysis.analysis.workflows[0];+const stage = workflow.stages[0];+const close = () => fireEvent.click(screen.getByRole("button", { name: `Close recording for ${stage.name}` }));+const play = () => fireEvent.click(screen.getByRole("button", { name: `View recording for ${stage.name}` }));+afterEach(cleanup);++describe("step evidence media", () => {+ it("loads only on request and releases the media when closed", async () => {+ const load = vi.fn().mockResolvedValue({ kind: "video", url: "https://fixture.invalid/clip.mp4", timestamp: stage.evidence[0].timestamp, frameId: 1, offsetSeconds: 2 });+ const release = vi.fn().mockResolvedValue(undefined);+ render(<WorkflowStepEvidence workflow={workflow} stage={stage} platform={{ loadWorkflowRecording: load, releaseWorkflowRecording: release } as unknown as WorkflowsPlatform} />);+ expect(load).not.toHaveBeenCalled();+ play();+ const video = await screen.findByLabelText("Local recording");+ expect(video).toHaveAttribute("controls");+ Object.defineProperty(video, "duration", { value: 30 });+ fireEvent.loadedMetadata(video);+ expect((video as HTMLVideoElement).currentTime).toBe(2);+ fireEvent.loadedData(video);+ close();+ await waitFor(() => expect(release).toHaveBeenCalledWith("https://fixture.invalid/clip.mp4"));+ expect(screen.queryByLabelText("Local recording")).toBeNull();+ });++ it("shows a failed load and retries on request", async () => {+ const load = vi.fn().mockRejectedValueOnce(new Error("offline")).mockResolvedValueOnce(null);+ render(<WorkflowStepEvidence workflow={workflow} stage={stage} platform={{ loadWorkflowRecording: load } as unknown as WorkflowsPlatform} />);+ play();+ expect(await screen.findByRole("alert")).toHaveTextContent("Recording unavailable");+ fireEvent.click(screen.getByRole("button", { name: "Retry" }));+ await waitFor(() => expect(load).toHaveBeenCalledTimes(2));+ await waitFor(() => expect(screen.queryByRole("alert")).toBeNull());+ });++ it("does not fabricate screenshots or offer playback without a source", () => {+ render(<WorkflowStepEvidence workflow={workflow} stage={{ ...stage, screenshot: null, evidence: [], procedure: [] }} platform={{ loadWorkflowRecording: vi.fn() } as unknown as WorkflowsPlatform} />);+ expect(screen.queryByRole("img")).toBeNull();+ expect(screen.queryByRole("button", { name: /View recording/ })).toBeNull();+ });++ it("ignores a response arriving after the player is closed", async () => {+ let resolve!: (value: unknown) => void;+ const load = vi.fn().mockReturnValue(new Promise(done => { resolve = done; }));+ const release = vi.fn().mockResolvedValue(undefined);+ render(<WorkflowStepEvidence workflow={workflow} stage={stage} platform={{ loadWorkflowRecording: load, releaseWorkflowRecording: release } as unknown as WorkflowsPlatform} />);+ play(); close();+ resolve({ kind: "video", url: "https://fixture.invalid/late.mp4" });+ await waitFor(() => expect(release).toHaveBeenCalledWith("https://fixture.invalid/late.mp4"));+ expect(screen.queryByLabelText("Local recording")).toBeNull();+ });+});diff --git a/apps/screenpipe-workflows-web/scripts/eval-workflow-editor.cjs b/apps/screenpipe-workflows-web/scripts/eval-workflow-editor.cjsindex 4e0282af1..b60c1cb80 100644--- a/apps/screenpipe-workflows-web/scripts/eval-workflow-editor.cjs+++ b/apps/screenpipe-workflows-web/scripts/eval-workflow-editor.cjs@@ -44,13 +44,27 @@ require("node:fs").mkdirSync(out, { recursive: true });await page.waitForTimeout(150);assert.equal(await button("Move step 2").evaluate(e => getComputedStyle(e).opacity), "1");await field("Workflow title").focus();- await page.getByText("Sources and screenshots", { exact: true }).first().click();- await page.locator("p").filter({ hasText: "Collect sources was observed in this fictional workspace." }).waitFor();- await page.getByText("Sources and screenshots", { exact: true }).first().click();+ const references = page.getByRole("region", { name: "References for Collect sources", exact: true });+ await references.getByText("1 source", { exact: true }).click();+ const raw = references.getByRole("region", { name: "Captured text 1 for Collect sources", exact: true });+ assert.equal(await raw.isVisible(), false);+ await references.getByText("Captured text", { exact: true }).click();+ await raw.waitFor();+ await references.getByText("1 source", { exact: true }).click();+ const enlarge = references.getByRole("button", { name: "Enlarge screenshot for Collect sources" });+ const smallWidth = (await enlarge.boundingBox()).width;+ await enlarge.click();+ const reduce = references.getByRole("button", { name: "Reduce screenshot for Collect sources" });+ assert((await reduce.boundingBox()).width > smallWidth);+ await page.screenshot({ path: out + "/screenshot-expanded.png" });+ await reduce.click();+ await references.getByRole("button", { name: "View recording for Collect sources" }).click();+ await references.getByRole("img", { name: "Captured moment for Collect sources", exact: true }).waitFor();+ await references.getByRole("button", { name: "Close recording for Collect sources" }).click();});await check("drag step, auto-save and retain its screenshot", async () => {await page.getByRole("article", { name: "Step 1", exact: true }).hover();- await button("Move step 1").dragTo(page.getByRole("article", { name: "Step 2", exact: true }));+ await button("Move step 1").dragTo(field("Step 2 title"));assert.equal(await field("Step 2 title").inputValue(), "Collect sources");await saved();const workflow = await stored();@@ -74,9 +88,13 @@ require("node:fs").mkdirSync(out, { recursive: true });const after = await stored();assert.equal(after.stages[2].name, "Collect sources");assert.deepEqual(after.stages[2].screenshot, before.stages[0].screenshot);- await page.getByRole("article", { name: "Step 3", exact: true }).hover();- await button("Move step 3").dragTo(page.getByRole("article", { name: "Step 1", exact: true }));+ await button("Move step 3").focus();+ await page.keyboard.press("Alt+ArrowUp");await saved();+ await button("Move step 2").focus();+ await page.keyboard.press("Alt+ArrowUp");+ await saved();+ assert.equal((await stored()).stages[0].name, "Collect sources");});await check("typing auto-saves without losing focus", async () => {await field("Workflow title").fill("Research brief");@@ -179,14 +197,25 @@ require("node:fs").mkdirSync(out, { recursive: true });const stage = catalog.analysis.workflows[0].stages[0];stage.procedure = [];stage.screenshot = null;+ stage.evidence[0].detail = "Fictional capture: navigation toolbar sidebar unread messages. ".repeat(700);const context = await browser.newContext();await context.addInitScript(data => localStorage.setItem("screenpipe:fictional-workflow-editor-preview", JSON.stringify(data)), catalog);+ context.setDefaultTimeout(10000);const legacy = await context.newPage();await legacy.goto(process.env.WORKFLOWS_PREVIEW_URL || "http://127.0.0.1:1431/preview");await legacy.getByRole("button", { name: "Open map", exact: true }).first().click();- await legacy.getByText("Sources and screenshots", { exact: true }).first().click();- await legacy.locator("p").filter({ hasText: stage.evidence[0].detail }).waitFor();- assert.equal(await legacy.getByRole("article", { name: "Step 1", exact: true }).getByText("No captured source attached.").count(), 0);+ const references = legacy.getByRole("region", { name: `References for ${stage.name}`, exact: true });+ await references.getByText("1 source", { exact: true }).click();+ const raw = references.getByRole("region", { name: `Captured text 1 for ${stage.name}`, exact: true });+ assert.equal(await raw.isVisible(), false);+ await references.getByText("Captured text", { exact: true }).click();+ assert.equal(await raw.textContent(), stage.evidence[0].detail);+ assert((await raw.boundingBox()).height <= 185);+ assert(await raw.evaluate(e => e.scrollHeight > e.clientHeight));+ await legacy.screenshot({ path: out + "/long-source.png" });+ await legacy.setViewportSize({ width: 390, height: 900 });+ await legacy.getByRole("button", { name: "Collapse left sidebar", exact: true }).click();+ assert(await raw.evaluate(e => e.getBoundingClientRect().right <= innerWidth));await context.close();});await check("touch layout keeps controls hidden until a step is focused", async () => {diff --git a/packages/workflows-ui/src/fixture-platform.ts b/packages/workflows-ui/src/fixture-platform.tsindex 19c4dc8ee..8d3993f95 100644--- a/packages/workflows-ui/src/fixture-platform.ts+++ b/packages/workflows-ui/src/fixture-platform.ts@@ -130,7 +130,7 @@ function fixtureWorkflow(input: FixtureWorkflow, index: number): WorkflowMap {observedOccurrences: 5 + index,observedDays: 4 + index,evidence: [allEvidence[stageIndex]],- screenshot: screenshot(name, input.apps[Math.min(stageIndex, input.apps.length - 1)], index * 10 + stageIndex + 1),+ screenshot: { ...screenshot(name, input.apps[Math.min(stageIndex, input.apps.length - 1)], index * 10 + stageIndex + 1), timestamp: allEvidence[stageIndex].timestamp, matchDistanceSeconds: 0 },})),bottlenecks: [{label: input.friction,@@ -384,7 +384,7 @@ function fixtureSkillReceipt(draft: WorkflowSkillDraft) {};}-async function rasterizeFixture(workflow: WorkflowMap) {+async function rasterizeFixture(workflow: Pick<WorkflowMap, "stages">) {// Rasterize this file's fictional SVGs so previews exercise the same raster-only export.for (const stage of workflow.stages) {if (!stage.screenshot?.dataUrl.startsWith("data:image/svg+xml")) continue;@@ -448,8 +448,14 @@ export function createFixtureWorkflowsPlatform(analysis: WorkflowAnalysis = fixtonField({ field: "company", value: "Preview company: a team that helps businesses resolve customer requests." });onField({ field: "summary", value: "Preview role: I manage customer operations and improve how our team handles requests." });},+ loadWorkflowRecording: async (timestamp, app) => {+ const stage = current.analysis.workflows.flatMap(w => w.stages).find(s => s.evidence.some(e => e.timestamp === timestamp && e.app === app));+ if (!stage?.screenshot) return null;+ await rasterizeFixture({ stages: [stage] });+ return { kind: "image", url: stage.screenshot.dataUrl, frameId: stage.screenshot.frameId, timestamp: stage.screenshot.timestamp, offsetSeconds: 0, matchDistanceSeconds: 0 };+ },ensureRuntime: async () => fixtureWorkflowRuntime,- loadCapturedWork: async () => restore(),+ loadCapturedWork: async () => { const value = restore(); await Promise.all(value.analysis.workflows.map(rasterizeFixture)); return structuredClone(value); },analyzeCapturedWork: async () => restore(),saveWorkflowEdits: async (draft) => {restore();diff --git a/packages/workflows-ui/src/workflow-editor.module.css b/packages/workflows-ui/src/workflow-editor.module.cssindex 47022c1fe..454f13d0e 100644--- a/packages/workflows-ui/src/workflow-editor.module.css+++ b/packages/workflows-ui/src/workflow-editor.module.css@@ -39,7 +39,7 @@.stepsHeading { margin: 18px 0 8px; padding: 0 5px; }.stepsHeading strong { color: var(--muted); font-size: 11px; font-weight: 500; }.steps { margin: 0 0 4px; padding-left: 26px; }-.step { position: relative; padding: 4px 0; margin: 0 0 12px; border: 0; background: transparent; }+.step { position: relative; padding: 4px 0; margin: 0 0 24px; border: 0; background: transparent; }.step.drop { box-shadow: 0 -2px 0 var(--ink); }.stepTop { padding-right: 30px; position: relative; display: grid; grid-template-columns: 24px minmax(0, 1fr) auto; align-items: start; }.number { padding-top: 7px; color: var(--muted); font-size: 12px; font-variant-numeric: tabular-nums; }diff --git a/packages/workflows-ui/src/workflow-step-evidence.module.css b/packages/workflows-ui/src/workflow-step-evidence.module.cssnew file mode 100644index 000000000..ef21d3c5a--- /dev/null+++ b/packages/workflows-ui/src/workflow-step-evidence.module.css@@ -0,0 +1,29 @@+/* screenpipe — AI that knows everything you've seen, said, or heard */+/* https://screenpipe.com */+.evidence { margin: 12px 0 4px; min-width: 0; }+.figure { max-width: 360px; margin: 0 0 6px; }+.figure.expanded { max-width: 100%; }+.figure .imageButton { display: block; padding: 0; position: relative; width: 100%; border: 1px solid var(--line); border-radius: 6px; overflow: hidden; background: var(--soft); cursor: zoom-in; }+.expanded .imageButton { cursor: zoom-out; }+.imageButton img { display: block; width: 100%; max-height: 520px; object-fit: contain; }+.zoom { position: absolute; right: 8px; bottom: 8px; display: grid; place-items: center; padding: 6px; background: var(--panel); border: 1px solid var(--line); border-radius: 4px; opacity: 0; }+.imageButton:hover .zoom, .imageButton:focus-visible .zoom { opacity: 1; }+.figure figcaption { display: flex; justify-content: space-between; align-items: center; gap: 8px; font-size: 11px; line-height: 1.5; color: var(--muted); }+.figure figcaption > span { min-width: 0; overflow-wrap: anywhere; }+.links { display: flex; flex-wrap: wrap; align-items: flex-start; gap: 4px 12px; font-size: 11px; color: var(--muted); }+.links > button { font-size: 11px; }+.sources { flex: 1; min-width: min(240px, 100%); }+.evidence summary { display: flex; align-items: center; gap: 6px; width: fit-content; min-height: 28px; list-style: none; cursor: pointer; font-size: 11px; color: var(--muted); }+.evidence summary::-webkit-details-marker { display: none; }+.evidence summary:focus-visible { outline: 2px solid var(--ink); outline-offset: 2px; }+.evidence details[open] > summary > svg { transform: rotate(180deg); }+.sourceList { max-height: 320px; overflow: auto; overscroll-behavior: contain; border-left: 1px solid var(--line); margin: 4px 0 8px; padding-left: 12px; }+.rawSource > summary { width: 100%; justify-content: space-between; flex-wrap: wrap; padding: 4px 0; }+.rawSource > summary > span:last-child { display: inline-flex; gap: 4px; align-items: center; }+.rawText { max-height: 160px; overflow: auto; overscroll-behavior: contain; white-space: pre-wrap; overflow-wrap: anywhere; font-size: 12px; line-height: 1.6; padding: 10px 12px; background: var(--soft); border-radius: 4px; color: var(--muted); }+.questions { font-size: 12px; color: var(--muted); }+.questions ul { padding-left: 18px; line-height: 1.6; }+.player { position: relative; padding-top: 22px; }+.player > section { margin: 0 0 8px; border-radius: 6px; }+.player .close { position: absolute; right: 0; top: -6px; }+@media (max-width: 560px) { .figure { max-width: 100%; } }diff --git a/packages/workflows-ui/src/workflow-step-evidence.tsx b/packages/workflows-ui/src/workflow-step-evidence.tsxnew file mode 100644index 000000000..4d092becf--- /dev/null+++ b/packages/workflows-ui/src/workflow-step-evidence.tsx@@ -0,0 +1,52 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+"use client";++import { useMemo, useState } from "react";+import { ChevronDown, Maximize2, Minimize2, Play, X } from "lucide-react";+import type { WorkflowMap, WorkflowStage } from "./model";+import type { WorkflowsPlatform } from "./platform";+import { CapturedMomentButton, WorkflowReplay } from "./workflow-replay";+import styles from "./workflow-step-evidence.module.css";++const when = (value: string) => Number.isFinite(Date.parse(value))+ ? new Date(value).toLocaleString(undefined, { month: "short", day: "numeric", hour: "numeric", minute: "2-digit" }) : "Date unavailable";++export function WorkflowStepEvidence({ workflow, stage, platform }: {+ workflow: WorkflowMap; stage: WorkflowStage; platform: WorkflowsPlatform;+}) {+ const [expanded, setExpanded] = useState(false);+ const [playing, setPlaying] = useState(false);+ const replay = useMemo(() => ({ ...workflow, stages: [stage] }), [workflow, stage]);+ const screenshot = stage.screenshot?.visualVerified ? stage.screenshot : null;+ const sources = [...(stage.procedure ?? []).filter(p => p.quote).map(p => ({ text: p.quote, timestamp: p.timestamp, app: p.app })),+ ...stage.evidence.map(e => ({ text: e.detail, timestamp: e.timestamp, app: e.app }))]+ .filter((entry, index, all) => entry.text && all.findIndex(e => e.text === entry.text && e.timestamp === entry.timestamp && e.app === entry.app) === index);+ const canReplay = platform.loadWorkflowRecording && stage.evidence.some(e => !["audio", "meeting"].includes(e.source ?? "") && Number.isFinite(Date.parse(e.timestamp)));+ return <section className={`${styles.evidence} ph-no-capture ph-mask`} aria-label={`References for ${stage.name}`}>+ {screenshot && !playing && <figure className={`${styles.figure} ${expanded ? styles.expanded : ""}`}>+ <button className={styles.imageButton} type="button" aria-label={`${expanded ? "Reduce" : "Enlarge"} screenshot for ${stage.name}`} aria-expanded={expanded} onClick={() => setExpanded(!expanded)}>+ <img src={screenshot.dataUrl} alt={`Captured reference for ${stage.name}`} loading="lazy" draggable={false} data-lm-disable="true" />+ <span className={styles.zoom}>{expanded ? <Minimize2 size={15} /> : <Maximize2 size={15} />}</span>+ </button>+ <figcaption><span>{screenshot.app} · {when(screenshot.timestamp)}</span><CapturedMomentButton compact frameId={screenshot.frameId} timestamp={screenshot.timestamp} open={platform.openCapturedMoment} /></figcaption>+ </figure>}+ {playing && <div className={styles.player}>+ <button type="button" className={styles.close} aria-label={`Close recording for ${stage.name}`} title="Close recording" onClick={() => setPlaying(false)}><X size={15} /></button>+ <WorkflowReplay workflow={replay} loadRecording={platform.loadWorkflowRecording} releaseRecording={platform.releaseWorkflowRecording} openCapturedMoment={platform.openCapturedMoment} />+ </div>}+ <div className={styles.links}>+ {canReplay && !playing && <button type="button" onClick={() => setPlaying(true)} aria-label={`View recording for ${stage.name}`}><Play size={13} />View recording</button>}+ {!!sources.length && <details className={styles.sources}>+ <summary>{sources.length} source{sources.length === 1 ? "" : "s"}<ChevronDown size={12} /></summary>+ <div className={styles.sourceList}>+ {sources.map((source, index) => <details className={styles.rawSource} key={`${source.timestamp}-${index}`}>+ <summary><span>{source.app || "Captured source"} · {when(source.timestamp)}</span><span>Captured text<ChevronDown size={12} /></span></summary>+ <div className={styles.rawText} role="region" aria-label={`Captured text ${index + 1} for ${stage.name}`} tabIndex={0}>{source.text}</div>+ </details>)}+ </div>+ </details>}+ </div>+ {!!stage.openQuestions?.length && <details className={styles.questions}><summary>Unresolved details<ChevronDown size={12} /></summary><ul>{stage.openQuestions.map(q => <li key={q}>{q}</li>)}</ul></details>}+ </section>;+}diff --git a/packages/workflows-ui/src/workflows-app.tsx b/packages/workflows-ui/src/workflows-app.tsxindex 7d10a37da..5aa84c530 100644--- a/packages/workflows-ui/src/workflows-app.tsx+++ b/packages/workflows-ui/src/workflows-app.tsx@@ -4,6 +4,7 @@"use client";import { useNavigationWidth } from "./use-navigation-width";+import { WorkflowStepEvidence } from "./workflow-step-evidence";import { WorkflowEditor } from "./workflow-editor";import { retainNewerWorkflowEdits, type WorkflowEdit } from "./workflow-edits";import { WorkflowModelControl } from "./model-choice";@@ -947,15 +948,7 @@ function WorkflowDetail({ workflow, navigate, platform, workProfile, saveCorrectreturn (<><button className={styles.backButton} onClick={() => navigate("workflows")}><ArrowLeft size={14} />All workflows</button>- {saveEdits ? <WorkflowEditor key={workflow.id || workflow.title} workflow={workflow} save={saveEdits} actions={workflowActions} renderSource={stage => <details className={styles.workflowQuality}>- <summary><div><strong>Sources and screenshots</strong></div><ChevronDown size={14} /></summary>- <p>{stage.apps.join(" · ")} · {stage.observedOccurrences} observations across {stage.observedDays} days</p>- {stage.screenshot?.visualVerified && <div className={styles.stageScreenshot}><div className={styles.screenshotFrame}><img src={stage.screenshot.dataUrl} alt={`Captured reference for ${stage.name}`} draggable={false} data-lm-disable="true" /></div><CapturedMomentButton frameId={stage.screenshot.frameId} timestamp={stage.screenshot.timestamp} open={platform.openCapturedMoment} /></div>}- {stage.procedure?.filter(p => p.quote).map((p, i) => <blockquote key={i}>{p.quote}<small>{p.app} · {formatEvidenceTimestamp(p.timestamp)}</small></blockquote>)}- {stage.evidence.filter(e => !stage.procedure?.some(p => p.quote === e.detail && p.timestamp === e.timestamp)).map((e, i) => <p key={`evidence-${i}`}>{e.detail}<small>{e.app} · {formatEvidenceTimestamp(e.timestamp)}</small></p>)}- {stage.openQuestions?.map((question, i) => <p key={i}>{question}</p>)}- {!stage.screenshot?.visualVerified && !stage.procedure?.some(p => p.quote) && !stage.evidence.length && <p>No captured source attached.</p>}- </details>} /> : <section className={styles.detailHeader}>+ {saveEdits ? <WorkflowEditor key={workflow.id || workflow.title} workflow={workflow} save={saveEdits} actions={workflowActions} renderSource={stage => <WorkflowStepEvidence workflow={workflow} stage={stage} platform={platform} />} /> : <section className={styles.detailHeader}><div><Pill>Evidence on {workflow.repetitions} captured day{workflow.repetitions === 1 ? "" : "s"}</Pill><h1>{workflow.title}</h1><p>{workflow.description}</p>{workflowActions}</div>{timing ? <div className={styles.detailTotal}><span>{timing.sampleCount > 1 ? "Avg. time / run" : "Time for one run"}</span><strong>{formatEstimatedMinutes(timing.averageMinutes)}</strong><small>{timing.sampleCount} run{timing.sampleCount === 1 ? "" : "s"} · estimated elapsed time</small></div> : measuredDuration && <div className={styles.detailTotal}><span>Observed meeting duration</span><strong>{workflowDurationLabel(workflow)}</strong></div>}</section>}