YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(workflows): let users review existing SOP screenshots
louisb0a6e43c4876
diff --git a/apps/screenpipe-app-tauri/app/dev/workflows-ux/page.tsx b/apps/screenpipe-app-tauri/app/dev/workflows-ux/page.tsxindex 11d8916a9..eb1976237 100644--- a/apps/screenpipe-app-tauri/app/dev/workflows-ux/page.tsx+++ b/apps/screenpipe-app-tauri/app/dev/workflows-ux/page.tsx@@ -12,6 +12,14 @@ import { ProductSwitcher } from "@/components/workflows/product-switcher";const previewTasks = { load: async () => ({ enabled: false, title: "Keep your workflows current", schedule: "every 24h" }), enable: async () => {} };const fixture = createFixtureWorkflowsPlatform();+// Older saved workflows have image bytes but no visual-verification metadata.+const legacyAnalysis = structuredClone(fixtureWorkflowAnalysis);+for (const workflow of legacyAnalysis.analysis.workflows) {+ for (const stage of workflow.stages) {+ if (stage.screenshot) delete stage.screenshot.visualVerified;+ }+}+const legacyFixture = createFixtureWorkflowsPlatform(legacyAnalysis);// Fictional receipt for the installed-state design only; no filesystem calls.const installedSkillFixture = { ...fixture, skillInstallMode: "local" as const,saveWorkflowSkill: async (draft: WorkflowSkillDraft) => ({ name: draft.name, path: `/example/skills/${draft.name}/SKILL.md`, updated: false,@@ -36,17 +44,17 @@ const processingFixture = { ...fixture, managesAnalysis: true,// This route never enables fixtures in a packaged application.export default function WorkflowUxPreview() {const [accessRequested, setAccessRequested] = useState(false);- const [screen, setScreen] = useState<"onboarding" | "empty" | "catalog" | "processing" | "tasks" | "skill" | "allowance" | "business">("processing");+ const [screen, setScreen] = useState<"onboarding" | "empty" | "catalog" | "processing" | "tasks" | "skill" | "allowance" | "business" | "legacy">("processing");if (process.env.NEXT_PUBLIC_SCREENPIPE_WEB_DEV !== "mock") return null;return <div className="h-screen bg-background text-foreground"><nav aria-label="Preview states" className="flex h-10 items-center justify-between gap-4 border-b border-border px-4 text-xs"><span className="text-muted-foreground">UX preview · fictional data</span>- <div className="flex gap-4"><button onClick={() => setScreen("allowance")}>Low allowance</button><button onClick={() => setScreen("business")}>Business access</button><button onClick={() => setScreen("skill")}>Skill install result</button><button onClick={() => setScreen("tasks")}>Enable tasks</button><button onClick={() => setScreen("onboarding")}>First task</button><button onClick={() => setScreen("empty")}>New history</button><button onClick={() => setScreen("catalog")}>Existing history</button><button onClick={() => setScreen("processing")}>Processing preview</button></div>+ <div className="flex gap-4"><button onClick={() => setScreen("legacy")}>SOP screenshots</button><button onClick={() => setScreen("allowance")}>Low allowance</button><button onClick={() => setScreen("business")}>Business access</button><button onClick={() => setScreen("skill")}>Skill install result</button><button onClick={() => setScreen("tasks")}>Enable tasks</button><button onClick={() => setScreen("onboarding")}>First task</button><button onClick={() => setScreen("empty")}>New history</button><button onClick={() => setScreen("catalog")}>Existing history</button><button onClick={() => setScreen("processing")}>Processing preview</button></div></nav><WorkflowTasksPrompt active={screen === "tasks"} tasks={previewTasks} /><div style={{height:"calc(100vh - 40px)", overflow:"auto"}}>{screen === "onboarding" ? <div className="flex min-h-full items-center"><FirstTaskChoice onComplete={async (mode) => { if(mode === "screenpipe") window.location.assign("/home?mode=screenpipe"); else setScreen("empty"); }} /></div>- : <WorkflowsApp onAnalysisUnavailable={() => setAccessRequested(true)} analysisUnavailableReason={screen === "allowance" || screen === "business" ? "Workflow updates are paused" : undefined} key={screen} statusNotice={screen === "allowance" || screen === "business" ? <WorkflowAccessNotice open={accessRequested} onOpenChange={setAccessRequested} access={screen === "allowance" ? {state:"paused",resetAt:undefined,message:"Workflow updates are paused to preserve AI allowance for chat."} : {state:"upgrade",message:"Automatic workflow discovery is included with Business. Your saved workflows remain available."}} onRetry={() => {}} onAccount={() => {}} onUsage={() => {}} /> : undefined} platform={screen === "processing" ? processingFixture : screen === "skill" ? installedSkillFixture : fixture} storageKey={null} initialAnalysis={(screen === "catalog" || screen === "tasks" || screen === "skill" || screen === "processing" || screen === "allowance" || screen === "business") ? fixtureWorkflowAnalysis : null} navigationBrand={<ProductSwitcher mode="workflows" onChange={mode => {if(mode === "screenpipe") window.location.assign("/home?mode=screenpipe");}} />} />}+ : <WorkflowsApp onAnalysisUnavailable={() => setAccessRequested(true)} analysisUnavailableReason={screen === "allowance" || screen === "business" ? "Workflow updates are paused" : undefined} key={screen} statusNotice={screen === "allowance" || screen === "business" ? <WorkflowAccessNotice open={accessRequested} onOpenChange={setAccessRequested} access={screen === "allowance" ? {state:"paused",resetAt:undefined,message:"Workflow updates are paused to preserve AI allowance for chat."} : {state:"upgrade",message:"Automatic workflow discovery is included with Business. Your saved workflows remain available."}} onRetry={() => {}} onAccount={() => {}} onUsage={() => {}} /> : undefined} platform={screen === "legacy" ? legacyFixture : screen === "processing" ? processingFixture : screen === "skill" ? installedSkillFixture : fixture} storageKey={null} initialAnalysis={screen === "legacy" ? legacyAnalysis : (screen === "catalog" || screen === "tasks" || screen === "skill" || screen === "processing" || screen === "allowance" || screen === "business") ? fixtureWorkflowAnalysis : null} navigationBrand={<ProductSwitcher mode="workflows" onChange={mode => {if(mode === "screenpipe") window.location.assign("/home?mode=screenpipe");}} />} />}</div></div>;}diff --git a/apps/screenpipe-app-tauri/lib/workflows/guide-storage.test.ts b/apps/screenpipe-app-tauri/lib/workflows/guide-storage.test.tsindex e2b865b74..5e5bd775c 100644--- a/apps/screenpipe-app-tauri/lib/workflows/guide-storage.test.ts+++ b/apps/screenpipe-app-tauri/lib/workflows/guide-storage.test.ts@@ -72,3 +72,18 @@ it("preserves unreadable drafts instead of replacing them", async () => {await expect(saveGuideToDisk(guide)).rejects.toThrow("unreadable");expect([...files.values()]).toEqual(["broken", "also broken"]);});++it("keeps screenshot review through disk validation and backup recovery", async () => {+ const reviewed = structuredClone(guide);+ reviewed.steps[0] = {+ ...reviewed.steps[0],+ sourceStage: 0,+ includeImage: true,+ imageReview: { frameId: 3090, timestamp: "2026-09-11T10:00:00Z" },+ };+ await saveGuideToDisk(reviewed);+ expect(await loadGuideFromDisk("research")).toEqual(reviewed);+ await saveGuideToDisk({ ...reviewed, title: "Updated" });+ files.set("workflows/guides.json", "broken");+ expect(await loadGuideFromDisk("research")).toEqual(reviewed);+});diff --git a/apps/screenpipe-app-tauri/lib/workflows/guide.test.tsx b/apps/screenpipe-app-tauri/lib/workflows/guide.test.tsxindex 761075753..e46d8a02f 100644--- a/apps/screenpipe-app-tauri/lib/workflows/guide.test.tsx+++ b/apps/screenpipe-app-tauri/lib/workflows/guide.test.tsx@@ -17,6 +17,7 @@ import {} from "../../../../packages/workflows-ui/src/guide";import { WorkflowGuide } from "../../../../packages/workflows-ui/src/workflow-guide";import { fixtureWorkflowAnalysis } from "../../../../packages/workflows-ui/src/fixture-platform";+import { sanitizeWorkflowAnalysis } from "../../../../packages/workflows-ui/src/catalog";const workflow = structuredClone(fixtureWorkflowAnalysis.analysis.workflows[0]);workflow.id = "research";workflow.revision = 3;@@ -103,6 +104,41 @@ describe("guide contracts and export", () => {"<img ",);});+ it("keeps older images through catalog loading without counting them as verified evidence", () => {+ const catalog = structuredClone(fixtureWorkflowAnalysis);+ catalog.analysis.workflows = [catalog.analysis.workflows[0]];+ const stage = catalog.analysis.workflows[0].stages[0];+ stage.screenshot = {+ ...workflow.stages[0].screenshot!,+ visualVerified: undefined,+ };+ const clean = sanitizeWorkflowAnalysis(catalog);+ expect(clean.analysis.workflows[0].stages[0].screenshot).toEqual(+ stage.screenshot,+ );+ expect(clean.analysis.workflows[0].quality.screenshotCount).toBe(2);+ });+ it("persists human review, but never accepts it from generated output or for another source", () => {+ const w = structuredClone(workflow);+ delete w.stages[0].screenshot!.visualVerified;+ const reviewed = structuredClone(guide);+ reviewed.steps[0].imageReview = {+ frameId: 1,+ timestamp: w.stages[0].screenshot!.timestamp,+ };+ const reopened = parseGuide(JSON.parse(JSON.stringify(reviewed)));+ expect(guideHtml(reopened, w, true)).toContain("<img ");+ expect(guideHtml(reopened, w, false)).not.toContain("<img ");+ expect(guideHtml(parseGuide(reviewed, w), w, true)).not.toContain("<img ");+ w.stages[0].screenshot!.frameId = 2;+ expect(guideHtml(reopened, w, true)).not.toContain("<img ");+ w.stages[0].screenshot!.frameId = 1;+ expect(guideHtml(reopened, { ...w, revision: 4 }, true)).not.toContain(+ "<img ",+ );+ w.stages[0].screenshot!.dataUrl = "https://example.com/tracker";+ expect(guideHtml(reopened, w, true)).not.toContain("<img ");+ });it("keeps image payloads and local paths out of the agent prompt", () => {expect(guidePrompt(workflow)).not.toContain("data:image");expect(guidePrompt(workflow)).toContain("Do not execute the workflow");@@ -142,6 +178,69 @@ describe("guide editor", () => {),);});+ it("lets an older saved SOP review and include a local screenshot without regenerating", async () => {+ const w = structuredClone(workflow);+ delete w.stages[0].screenshot!.visualVerified;+ w.stages[1].screenshot = undefined;+ const oldGuide = structuredClone(guide);+ oldGuide.steps[0].includeImage = false;+ const platform = { ...host(), load: vi.fn(async () => oldGuide) };+ const view = render(+ <WorkflowGuide workflow={w} platform={platform} close={() => {}} />,+ );+ await screen.findByRole("heading", { name: "Research guide" });+ expect(screen.queryByRole("img")).toBeNull();+ fireEvent.click(screen.getByRole("button", { name: "Review screenshot" }));+ const preview = screen.getByRole("img", {+ name: "Review source for Collect sources",+ });+ const include = screen.getByRole("button", { name: "Include screenshot" });+ expect(include).toBeDisabled();+ fireEvent.load(preview);+ fireEvent.click(include);+ await screen.findByRole("img", { name: "Source for Collect sources" });+ await waitFor(() => expect(platform.save).toHaveBeenCalled());+ const saved = parseGuide(+ JSON.parse(JSON.stringify(platform.save.mock.calls.at(-1)![0])),+ );+ expect(saved.steps[0].includeImage).toBe(true);+ expect(saved.steps[0].imageReview?.frameId).toBe(1);+ view.unmount();+ render(+ <WorkflowGuide+ workflow={w}+ platform={{ ...platform, load: async () => saved }}+ close={() => {}}+ />,+ );+ await screen.findByRole("img", { name: "Source for Collect sources" });+ expect(platform.generate).not.toHaveBeenCalled();+ fireEvent.click(screen.getByRole("button", { name: "Edit SOP" }));+ fireEvent.click(screen.getByRole("button", { name: "Remove screenshot" }));+ await waitFor(() =>+ expect(platform.save.mock.calls.at(-1)![0].steps[0].includeImage).toBe(+ false,+ ),+ );+ });+ it("does not include a screenshot that fails to load", async () => {+ const w = structuredClone(workflow);+ delete w.stages[0].screenshot!.visualVerified;+ w.stages[1].screenshot = undefined;+ render(<WorkflowGuide workflow={w} platform={host()} close={() => {}} />);+ fireEvent.click(+ await screen.findByRole("button", { name: "Review screenshot" }),+ );+ fireEvent.error(+ screen.getByRole("img", { name: "Review source for Collect sources" }),+ );+ expect(+ screen.getByRole("button", { name: "Include screenshot" }),+ ).toBeDisabled();+ expect(+ screen.getByText("This screenshot could not be loaded."),+ ).toBeInTheDocument();+ });it("scrolls guide sections without changing the host route", async () => {const scroll = vi.fn();render(diff --git a/packages/workflows-ui/src/catalog.ts b/packages/workflows-ui/src/catalog.tsindex aff4c9e30..125643707 100644--- a/packages/workflows-ui/src/catalog.ts+++ b/packages/workflows-ui/src/catalog.ts@@ -144,7 +144,8 @@ export function sanitizeWorkflowAnalysis(analysis: WorkflowAnalysis): WorkflowAnfrequency: `Observed on ${workflow.quality.distinctDays} captured day${workflow.quality.distinctDays === 1 ? "" : "s"} in a ${workflow.analysisDays}-day scan`,stages: workflow.stages.map((stage) => ({...stage,- screenshot: stage.screenshot?.visualVerified ? stage.screenshot : undefined,+ // Keep legacy image bytes for explicit SOP review. Evidence surfaces+ // and coverage counts still require visualVerified.activeMinutes: 0,waitingMinutes: 0,durationSource: "unknown" as const,diff --git a/packages/workflows-ui/src/guide.ts b/packages/workflows-ui/src/guide.tsindex 6034fdb90..9418da9ce 100644--- a/packages/workflows-ui/src/guide.ts+++ b/packages/workflows-ui/src/guide.ts@@ -15,6 +15,7 @@ export type WorkflowGuide = {expectedResult: string;sourceStage: number | null;includeImage: boolean;+ imageReview?: { frameId: number; timestamp: string };narration?: string;}>;exceptions: string[];@@ -54,6 +55,10 @@ export function parseGuide((s.narration === undefined ||(text(s.narration) && [...s.narration].length <= 800)) &&typeof s.includeImage === "boolean" &&+ (s.imageReview === undefined ||+ (Number.isInteger(s.imageReview?.frameId) &&+ s.imageReview.frameId >= 0 &&+ text(s.imageReview.timestamp))) &&(s.sourceStage === null ||(Number.isInteger(s.sourceStage) && s.sourceStage >= 0)),)@@ -89,6 +94,7 @@ export function parseGuide(sourceStage,includeImage,narration,+ imageReview,}) => ({title,instruction,@@ -96,6 +102,9 @@ export function parseGuide(sourceStage,includeImage,...(narration !== undefined ? { narration } : {}),+ // A workflow argument validates agent output. Only disk/UI drafts may+ // carry a human review; an agent cannot grant itself that approval.+ ...(!workflow && imageReview ? { imageReview } : {}),}),),exceptions: g.exceptions,@@ -117,17 +126,32 @@ Attached workflow evidence:${evidence}`;}-export function guideImage(+/** Existing local imagery is available for review even without legacy verification metadata. */+export function guideSourceImage(workflow: WorkflowMap,sourceStage: number | null,-): string | null {+) {const image =sourceStage === null ? null : workflow.stages[sourceStage]?.screenshot;- return image?.visualVerified &&+ return image &&/^data:image\/(png|jpeg|webp);base64,[A-Za-z0-9+/=]+$/.test(image.dataUrl)- ? image.dataUrl+ ? image: null;}++export function guideImage(+ workflow: WorkflowMap,+ sourceStage: number | null,+ review?: WorkflowGuide["steps"][number]["imageReview"],+): string | null {+ const image = guideSourceImage(workflow, sourceStage);+ const reviewed =+ image &&+ review &&+ review.frameId === image.frameId &&+ review.timestamp === image.timestamp;+ return image && (image.visualVerified || reviewed) ? image.dataUrl : null;+}const escape = (text: string) =>text.replace(/[&<>"']/g,@@ -153,7 +177,7 @@ export function guideHtml(includeImages &&guide.sourceRevision === (workflow.revision ?? 0) &&s.includeImage- ? guideImage(workflow, s.sourceStage)+ ? guideImage(workflow, s.sourceStage, s.imageReview): null;return `<article><h2>${i + 1}. ${escape(s.title)}</h2><p>${escape(s.instruction)}</p>${image ? `<img alt="${escape(s.title)}" src="${image}">` : ""}${s.expectedResult ? `<p><strong>Expected result:</strong> ${escape(s.expectedResult)}</p>` : ""}</article>`;})diff --git a/packages/workflows-ui/src/workflow-guide.module.css b/packages/workflows-ui/src/workflow-guide.module.cssindex df93614ca..0fecf1649 100644--- a/packages/workflows-ui/src/workflow-guide.module.css+++ b/packages/workflows-ui/src/workflow-guide.module.css@@ -225,16 +225,27 @@ label.result {display: block;margin-top: 18px;}-.noImage {+.imageReview {+ border: 1px solid var(--line, #dfe1d9);+ border-radius: 8px;+ overflow: hidden;+ margin: 20px 0;+}+.imageReviewHeader {display: flex;- gap: 9px;+ flex-wrap: wrap;+ gap: 12px;align-items: center;- min-height: 60px;- border: 1px dashed var(--line, #dfe1d9);- border-radius: 7px;+ justify-content: space-between;padding: 12px 16px;- font-size: 12px;color: var(--muted, #777b70);+ font-size: 13px;+}+.imageReview img {+ border: 0;+ border-top: 1px solid var(--line, #dfe1d9);+ border-bottom: 1px solid var(--line, #dfe1d9);+ border-radius: 0;}.guide input:not([type="checkbox"]),.guide textarea {diff --git a/packages/workflows-ui/src/workflow-guide.tsx b/packages/workflows-ui/src/workflow-guide.tsxindex ab757a3d2..d6ec4abf8 100644--- a/packages/workflows-ui/src/workflow-guide.tsx+++ b/packages/workflows-ui/src/workflow-guide.tsx@@ -19,7 +19,12 @@ import {} from "lucide-react";import type { WorkflowMap } from "./model";import type { WorkflowsPlatform } from "./platform";-import { guideHtml, guideImage, type WorkflowGuide as Guide } from "./guide";+import {+ guideHtml,+ guideImage,+ guideSourceImage,+ type WorkflowGuide as Guide,+} from "./guide";import { GuideAssistant } from "./guide-assistant";import styles from "./workflow-guide.module.css";@@ -379,8 +384,11 @@ export function WorkflowGuide({{draft.steps.map((step, i) => {const image =!stale && step.includeImage- ? guideImage(workflow, step.sourceStage)+ ? guideImage(workflow, step.sourceStage, step.imageReview): null;+ const source = !stale+ ? guideSourceImage(workflow, step.sourceStage)+ : null;return (<sectionclassName={styles.step}@@ -491,27 +499,35 @@ export function WorkflowGuide({</button>)}</figure>+ ) : source ? (+ <ScreenshotReview+ key={`${step.sourceStage}:${source.dataUrl}`}+ source={source}+ title={step.title}+ include={() => {+ update({+ ...draft,+ steps: draft.steps.map((s, j) =>+ j === i+ ? {+ ...s,+ includeImage: true,+ imageReview: {+ frameId: source.frameId,+ timestamp: source.timestamp,+ },+ }+ : s,+ ),+ });+ }}+ />) : (- <div className={styles.noImage}>- <ImageOff size={15} />- <span>No screenshot included</span>- {editing &&- !stale &&- guideImage(workflow, step.sourceStage) && (- <button- onClick={() =>- update({- ...draft,- steps: draft.steps.map((s, j) =>- j === i ? { ...s, includeImage: true } : s,- ),- })- }- >- Include source screenshot- </button>- )}- </div>+ <p className={styles.muted}>+ {stale+ ? "Source changed. Regenerate this SOP to review its screenshots."+ : "No captured screenshot for this step."}+ </p>)}{editing ? (<label className={styles.result}>@@ -638,3 +654,54 @@ export function WorkflowGuide({</div>);}++function ScreenshotReview({+ source,+ title,+ include,+}: {+ source: NonNullable<ReturnType<typeof guideSourceImage>>;+ title: string;+ include: () => void;+}) {+ const [open, setOpen] = useState(false);+ const [loaded, setLoaded] = useState(false);+ const [failed, setFailed] = useState(false);+ return (+ <div className={styles.imageReview}>+ <div className={styles.imageReviewHeader}>+ <span>Saved screenshot available</span>+ <button aria-expanded={open} onClick={() => setOpen(!open)}>+ {open ? "Hide screenshot" : "Review screenshot"}+ </button>+ </div>+ {open && (+ <>+ <img+ src={source.dataUrl}+ alt={`Review source for ${title}`}+ draggable={false}+ onLoad={() => {+ setLoaded(true);+ setFailed(false);+ }}+ onError={() => {+ setFailed(true);+ setLoaded(false);+ }}+ />+ <div className={styles.imageReviewHeader}>+ <span>+ {failed+ ? "This screenshot could not be loaded."+ : "Does this image show the step clearly?"}+ </span>+ <button disabled={!loaded || failed} onClick={include}>+ Include screenshot+ </button>+ </div>+ </>+ )}+ </div>+ );+}diff --git a/packages/workflows-ui/src/workflows-app.tsx b/packages/workflows-ui/src/workflows-app.tsxindex 07e0751f7..73ac21bb5 100644--- a/packages/workflows-ui/src/workflows-app.tsx+++ b/packages/workflows-ui/src/workflows-app.tsx@@ -957,7 +957,7 @@ function WorkflowDetail({ workflow, navigate, platform, workProfile, saveCorrect</li>)}</ol> : <p>Step not yet verified.</p>}{!!stage.openQuestions?.length && <details className={styles.procedureQuestions}><summary>Unresolved details</summary><ul>{stage.openQuestions.map((question) => <li key={question}>{question}</li>)}</ul></details>}</section>- {stage.screenshot && <div className={styles.stageScreenshot}>+ {stage.screenshot?.visualVerified && <div className={styles.stageScreenshot}><><div className={styles.screenshotFrame}><img src={stage.screenshot.dataUrl} alt={`Captured screen evidence for ${stage.name}`} draggable={false} data-lm-disable="true" /></div><div><Camera size={12} /><span>{formatEvidenceTimestamp(stage.screenshot.timestamp)} · {stage.screenshot.app} · {screenshotMatchLabel(stage.screenshot.matchDistanceSeconds)}</span><CapturedMomentButton frameId={stage.screenshot.frameId} timestamp={stage.screenshot.timestamp} open={platform.openCapturedMoment} /></div>