Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(workflows): let users review existing SOP screenshots

louisb0a6e43c4876
diff --git a/apps/screenpipe-app-tauri/app/dev/workflows-ux/page.tsx b/apps/screenpipe-app-tauri/app/dev/workflows-ux/page.tsx
index 11d8916a9..eb1976237 100644
--- a/apps/screenpipe-app-tauri/app/dev/workflows-ux/page.tsx
+++ b/apps/screenpipe-app-tauri/app/dev/workflows-ux/page.tsx
@@ -12,6 +12,14 @@ import { ProductSwitcher } from "@/components/workflows/product-switcher";
const previewTasks = { load: async () => ({ enabled: false, title: "Keep your workflows current", schedule: "every 24h" }), enable: async () => {} };
const fixture = createFixtureWorkflowsPlatform();
+// Older saved workflows have image bytes but no visual-verification metadata.
+const legacyAnalysis = structuredClone(fixtureWorkflowAnalysis);
+for (const workflow of legacyAnalysis.analysis.workflows) {
+ for (const stage of workflow.stages) {
+ if (stage.screenshot) delete stage.screenshot.visualVerified;
+ }
+}
+const legacyFixture = createFixtureWorkflowsPlatform(legacyAnalysis);
// Fictional receipt for the installed-state design only; no filesystem calls.
const installedSkillFixture = { ...fixture, skillInstallMode: "local" as const,
saveWorkflowSkill: async (draft: WorkflowSkillDraft) => ({ name: draft.name, path: `/example/skills/${draft.name}/SKILL.md`, updated: false,
@@ -36,17 +44,17 @@ const processingFixture = { ...fixture, managesAnalysis: true,
// This route never enables fixtures in a packaged application.
export default function WorkflowUxPreview() {
const [accessRequested, setAccessRequested] = useState(false);
- const [screen, setScreen] = useState<"onboarding" | "empty" | "catalog" | "processing" | "tasks" | "skill" | "allowance" | "business">("processing");
+ const [screen, setScreen] = useState<"onboarding" | "empty" | "catalog" | "processing" | "tasks" | "skill" | "allowance" | "business" | "legacy">("processing");
if (process.env.NEXT_PUBLIC_SCREENPIPE_WEB_DEV !== "mock") return null;
return <div className="h-screen bg-background text-foreground">
<nav aria-label="Preview states" className="flex h-10 items-center justify-between gap-4 border-b border-border px-4 text-xs">
<span className="text-muted-foreground">UX preview · fictional data</span>
- <div className="flex gap-4"><button onClick={() => setScreen("allowance")}>Low allowance</button><button onClick={() => setScreen("business")}>Business access</button><button onClick={() => setScreen("skill")}>Skill install result</button><button onClick={() => setScreen("tasks")}>Enable tasks</button><button onClick={() => setScreen("onboarding")}>First task</button><button onClick={() => setScreen("empty")}>New history</button><button onClick={() => setScreen("catalog")}>Existing history</button><button onClick={() => setScreen("processing")}>Processing preview</button></div>
+ <div className="flex gap-4"><button onClick={() => setScreen("legacy")}>SOP screenshots</button><button onClick={() => setScreen("allowance")}>Low allowance</button><button onClick={() => setScreen("business")}>Business access</button><button onClick={() => setScreen("skill")}>Skill install result</button><button onClick={() => setScreen("tasks")}>Enable tasks</button><button onClick={() => setScreen("onboarding")}>First task</button><button onClick={() => setScreen("empty")}>New history</button><button onClick={() => setScreen("catalog")}>Existing history</button><button onClick={() => setScreen("processing")}>Processing preview</button></div>
</nav>
<WorkflowTasksPrompt active={screen === "tasks"} tasks={previewTasks} />
<div style={{height:"calc(100vh - 40px)", overflow:"auto"}}>
{screen === "onboarding" ? <div className="flex min-h-full items-center"><FirstTaskChoice onComplete={async (mode) => { if(mode === "screenpipe") window.location.assign("/home?mode=screenpipe"); else setScreen("empty"); }} /></div>
- : <WorkflowsApp onAnalysisUnavailable={() => setAccessRequested(true)} analysisUnavailableReason={screen === "allowance" || screen === "business" ? "Workflow updates are paused" : undefined} key={screen} statusNotice={screen === "allowance" || screen === "business" ? <WorkflowAccessNotice open={accessRequested} onOpenChange={setAccessRequested} access={screen === "allowance" ? {state:"paused",resetAt:undefined,message:"Workflow updates are paused to preserve AI allowance for chat."} : {state:"upgrade",message:"Automatic workflow discovery is included with Business. Your saved workflows remain available."}} onRetry={() => {}} onAccount={() => {}} onUsage={() => {}} /> : undefined} platform={screen === "processing" ? processingFixture : screen === "skill" ? installedSkillFixture : fixture} storageKey={null} initialAnalysis={(screen === "catalog" || screen === "tasks" || screen === "skill" || screen === "processing" || screen === "allowance" || screen === "business") ? fixtureWorkflowAnalysis : null} navigationBrand={<ProductSwitcher mode="workflows" onChange={mode => {if(mode === "screenpipe") window.location.assign("/home?mode=screenpipe");}} />} />}
+ : <WorkflowsApp onAnalysisUnavailable={() => setAccessRequested(true)} analysisUnavailableReason={screen === "allowance" || screen === "business" ? "Workflow updates are paused" : undefined} key={screen} statusNotice={screen === "allowance" || screen === "business" ? <WorkflowAccessNotice open={accessRequested} onOpenChange={setAccessRequested} access={screen === "allowance" ? {state:"paused",resetAt:undefined,message:"Workflow updates are paused to preserve AI allowance for chat."} : {state:"upgrade",message:"Automatic workflow discovery is included with Business. Your saved workflows remain available."}} onRetry={() => {}} onAccount={() => {}} onUsage={() => {}} /> : undefined} platform={screen === "legacy" ? legacyFixture : screen === "processing" ? processingFixture : screen === "skill" ? installedSkillFixture : fixture} storageKey={null} initialAnalysis={screen === "legacy" ? legacyAnalysis : (screen === "catalog" || screen === "tasks" || screen === "skill" || screen === "processing" || screen === "allowance" || screen === "business") ? fixtureWorkflowAnalysis : null} navigationBrand={<ProductSwitcher mode="workflows" onChange={mode => {if(mode === "screenpipe") window.location.assign("/home?mode=screenpipe");}} />} />}
</div>
</div>;
}
diff --git a/apps/screenpipe-app-tauri/lib/workflows/guide-storage.test.ts b/apps/screenpipe-app-tauri/lib/workflows/guide-storage.test.ts
index e2b865b74..5e5bd775c 100644
--- a/apps/screenpipe-app-tauri/lib/workflows/guide-storage.test.ts
+++ b/apps/screenpipe-app-tauri/lib/workflows/guide-storage.test.ts
@@ -72,3 +72,18 @@ it("preserves unreadable drafts instead of replacing them", async () => {
await expect(saveGuideToDisk(guide)).rejects.toThrow("unreadable");
expect([...files.values()]).toEqual(["broken", "also broken"]);
});
+
+it("keeps screenshot review through disk validation and backup recovery", async () => {
+ const reviewed = structuredClone(guide);
+ reviewed.steps[0] = {
+ ...reviewed.steps[0],
+ sourceStage: 0,
+ includeImage: true,
+ imageReview: { frameId: 3090, timestamp: "2026-09-11T10:00:00Z" },
+ };
+ await saveGuideToDisk(reviewed);
+ expect(await loadGuideFromDisk("research")).toEqual(reviewed);
+ await saveGuideToDisk({ ...reviewed, title: "Updated" });
+ files.set("workflows/guides.json", "broken");
+ expect(await loadGuideFromDisk("research")).toEqual(reviewed);
+});
diff --git a/apps/screenpipe-app-tauri/lib/workflows/guide.test.tsx b/apps/screenpipe-app-tauri/lib/workflows/guide.test.tsx
index 761075753..e46d8a02f 100644
--- a/apps/screenpipe-app-tauri/lib/workflows/guide.test.tsx
+++ b/apps/screenpipe-app-tauri/lib/workflows/guide.test.tsx
@@ -17,6 +17,7 @@ import {
} from "../../../../packages/workflows-ui/src/guide";
import { WorkflowGuide } from "../../../../packages/workflows-ui/src/workflow-guide";
import { fixtureWorkflowAnalysis } from "../../../../packages/workflows-ui/src/fixture-platform";
+import { sanitizeWorkflowAnalysis } from "../../../../packages/workflows-ui/src/catalog";
const workflow = structuredClone(fixtureWorkflowAnalysis.analysis.workflows[0]);
workflow.id = "research";
workflow.revision = 3;
@@ -103,6 +104,41 @@ describe("guide contracts and export", () => {
"<img ",
);
});
+ it("keeps older images through catalog loading without counting them as verified evidence", () => {
+ const catalog = structuredClone(fixtureWorkflowAnalysis);
+ catalog.analysis.workflows = [catalog.analysis.workflows[0]];
+ const stage = catalog.analysis.workflows[0].stages[0];
+ stage.screenshot = {
+ ...workflow.stages[0].screenshot!,
+ visualVerified: undefined,
+ };
+ const clean = sanitizeWorkflowAnalysis(catalog);
+ expect(clean.analysis.workflows[0].stages[0].screenshot).toEqual(
+ stage.screenshot,
+ );
+ expect(clean.analysis.workflows[0].quality.screenshotCount).toBe(2);
+ });
+ it("persists human review, but never accepts it from generated output or for another source", () => {
+ const w = structuredClone(workflow);
+ delete w.stages[0].screenshot!.visualVerified;
+ const reviewed = structuredClone(guide);
+ reviewed.steps[0].imageReview = {
+ frameId: 1,
+ timestamp: w.stages[0].screenshot!.timestamp,
+ };
+ const reopened = parseGuide(JSON.parse(JSON.stringify(reviewed)));
+ expect(guideHtml(reopened, w, true)).toContain("<img ");
+ expect(guideHtml(reopened, w, false)).not.toContain("<img ");
+ expect(guideHtml(parseGuide(reviewed, w), w, true)).not.toContain("<img ");
+ w.stages[0].screenshot!.frameId = 2;
+ expect(guideHtml(reopened, w, true)).not.toContain("<img ");
+ w.stages[0].screenshot!.frameId = 1;
+ expect(guideHtml(reopened, { ...w, revision: 4 }, true)).not.toContain(
+ "<img ",
+ );
+ w.stages[0].screenshot!.dataUrl = "https://example.com/tracker";
+ expect(guideHtml(reopened, w, true)).not.toContain("<img ");
+ });
it("keeps image payloads and local paths out of the agent prompt", () => {
expect(guidePrompt(workflow)).not.toContain("data:image");
expect(guidePrompt(workflow)).toContain("Do not execute the workflow");
@@ -142,6 +178,69 @@ describe("guide editor", () => {
),
);
});
+ it("lets an older saved SOP review and include a local screenshot without regenerating", async () => {
+ const w = structuredClone(workflow);
+ delete w.stages[0].screenshot!.visualVerified;
+ w.stages[1].screenshot = undefined;
+ const oldGuide = structuredClone(guide);
+ oldGuide.steps[0].includeImage = false;
+ const platform = { ...host(), load: vi.fn(async () => oldGuide) };
+ const view = render(
+ <WorkflowGuide workflow={w} platform={platform} close={() => {}} />,
+ );
+ await screen.findByRole("heading", { name: "Research guide" });
+ expect(screen.queryByRole("img")).toBeNull();
+ fireEvent.click(screen.getByRole("button", { name: "Review screenshot" }));
+ const preview = screen.getByRole("img", {
+ name: "Review source for Collect sources",
+ });
+ const include = screen.getByRole("button", { name: "Include screenshot" });
+ expect(include).toBeDisabled();
+ fireEvent.load(preview);
+ fireEvent.click(include);
+ await screen.findByRole("img", { name: "Source for Collect sources" });
+ await waitFor(() => expect(platform.save).toHaveBeenCalled());
+ const saved = parseGuide(
+ JSON.parse(JSON.stringify(platform.save.mock.calls.at(-1)![0])),
+ );
+ expect(saved.steps[0].includeImage).toBe(true);
+ expect(saved.steps[0].imageReview?.frameId).toBe(1);
+ view.unmount();
+ render(
+ <WorkflowGuide
+ workflow={w}
+ platform={{ ...platform, load: async () => saved }}
+ close={() => {}}
+ />,
+ );
+ await screen.findByRole("img", { name: "Source for Collect sources" });
+ expect(platform.generate).not.toHaveBeenCalled();
+ fireEvent.click(screen.getByRole("button", { name: "Edit SOP" }));
+ fireEvent.click(screen.getByRole("button", { name: "Remove screenshot" }));
+ await waitFor(() =>
+ expect(platform.save.mock.calls.at(-1)![0].steps[0].includeImage).toBe(
+ false,
+ ),
+ );
+ });
+ it("does not include a screenshot that fails to load", async () => {
+ const w = structuredClone(workflow);
+ delete w.stages[0].screenshot!.visualVerified;
+ w.stages[1].screenshot = undefined;
+ render(<WorkflowGuide workflow={w} platform={host()} close={() => {}} />);
+ fireEvent.click(
+ await screen.findByRole("button", { name: "Review screenshot" }),
+ );
+ fireEvent.error(
+ screen.getByRole("img", { name: "Review source for Collect sources" }),
+ );
+ expect(
+ screen.getByRole("button", { name: "Include screenshot" }),
+ ).toBeDisabled();
+ expect(
+ screen.getByText("This screenshot could not be loaded."),
+ ).toBeInTheDocument();
+ });
it("scrolls guide sections without changing the host route", async () => {
const scroll = vi.fn();
render(
diff --git a/packages/workflows-ui/src/catalog.ts b/packages/workflows-ui/src/catalog.ts
index aff4c9e30..125643707 100644
--- a/packages/workflows-ui/src/catalog.ts
+++ b/packages/workflows-ui/src/catalog.ts
@@ -144,7 +144,8 @@ export function sanitizeWorkflowAnalysis(analysis: WorkflowAnalysis): WorkflowAn
frequency: `Observed on ${workflow.quality.distinctDays} captured day${workflow.quality.distinctDays === 1 ? "" : "s"} in a ${workflow.analysisDays}-day scan`,
stages: workflow.stages.map((stage) => ({
...stage,
- screenshot: stage.screenshot?.visualVerified ? stage.screenshot : undefined,
+ // Keep legacy image bytes for explicit SOP review. Evidence surfaces
+ // and coverage counts still require visualVerified.
activeMinutes: 0,
waitingMinutes: 0,
durationSource: "unknown" as const,
diff --git a/packages/workflows-ui/src/guide.ts b/packages/workflows-ui/src/guide.ts
index 6034fdb90..9418da9ce 100644
--- a/packages/workflows-ui/src/guide.ts
+++ b/packages/workflows-ui/src/guide.ts
@@ -15,6 +15,7 @@ export type WorkflowGuide = {
expectedResult: string;
sourceStage: number | null;
includeImage: boolean;
+ imageReview?: { frameId: number; timestamp: string };
narration?: string;
}>;
exceptions: string[];
@@ -54,6 +55,10 @@ export function parseGuide(
(s.narration === undefined ||
(text(s.narration) && [...s.narration].length <= 800)) &&
typeof s.includeImage === "boolean" &&
+ (s.imageReview === undefined ||
+ (Number.isInteger(s.imageReview?.frameId) &&
+ s.imageReview.frameId >= 0 &&
+ text(s.imageReview.timestamp))) &&
(s.sourceStage === null ||
(Number.isInteger(s.sourceStage) && s.sourceStage >= 0)),
)
@@ -89,6 +94,7 @@ export function parseGuide(
sourceStage,
includeImage,
narration,
+ imageReview,
}) => ({
title,
instruction,
@@ -96,6 +102,9 @@ export function parseGuide(
sourceStage,
includeImage,
...(narration !== undefined ? { narration } : {}),
+ // A workflow argument validates agent output. Only disk/UI drafts may
+ // carry a human review; an agent cannot grant itself that approval.
+ ...(!workflow && imageReview ? { imageReview } : {}),
}),
),
exceptions: g.exceptions,
@@ -117,17 +126,32 @@ Attached workflow evidence:
${evidence}`;
}
-export function guideImage(
+/** Existing local imagery is available for review even without legacy verification metadata. */
+export function guideSourceImage(
workflow: WorkflowMap,
sourceStage: number | null,
-): string | null {
+) {
const image =
sourceStage === null ? null : workflow.stages[sourceStage]?.screenshot;
- return image?.visualVerified &&
+ return image &&
/^data:image\/(png|jpeg|webp);base64,[A-Za-z0-9+/=]+$/.test(image.dataUrl)
- ? image.dataUrl
+ ? image
: null;
}
+
+export function guideImage(
+ workflow: WorkflowMap,
+ sourceStage: number | null,
+ review?: WorkflowGuide["steps"][number]["imageReview"],
+): string | null {
+ const image = guideSourceImage(workflow, sourceStage);
+ const reviewed =
+ image &&
+ review &&
+ review.frameId === image.frameId &&
+ review.timestamp === image.timestamp;
+ return image && (image.visualVerified || reviewed) ? image.dataUrl : null;
+}
const escape = (text: string) =>
text.replace(
/[&<>"']/g,
@@ -153,7 +177,7 @@ export function guideHtml(
includeImages &&
guide.sourceRevision === (workflow.revision ?? 0) &&
s.includeImage
- ? guideImage(workflow, s.sourceStage)
+ ? guideImage(workflow, s.sourceStage, s.imageReview)
: null;
return `<article><h2>${i + 1}. ${escape(s.title)}</h2><p>${escape(s.instruction)}</p>${image ? `<img alt="${escape(s.title)}" src="${image}">` : ""}${s.expectedResult ? `<p><strong>Expected result:</strong> ${escape(s.expectedResult)}</p>` : ""}</article>`;
})
diff --git a/packages/workflows-ui/src/workflow-guide.module.css b/packages/workflows-ui/src/workflow-guide.module.css
index df93614ca..0fecf1649 100644
--- a/packages/workflows-ui/src/workflow-guide.module.css
+++ b/packages/workflows-ui/src/workflow-guide.module.css
@@ -225,16 +225,27 @@ label.result {
display: block;
margin-top: 18px;
}
-.noImage {
+.imageReview {
+ border: 1px solid var(--line, #dfe1d9);
+ border-radius: 8px;
+ overflow: hidden;
+ margin: 20px 0;
+}
+.imageReviewHeader {
display: flex;
- gap: 9px;
+ flex-wrap: wrap;
+ gap: 12px;
align-items: center;
- min-height: 60px;
- border: 1px dashed var(--line, #dfe1d9);
- border-radius: 7px;
+ justify-content: space-between;
padding: 12px 16px;
- font-size: 12px;
color: var(--muted, #777b70);
+ font-size: 13px;
+}
+.imageReview img {
+ border: 0;
+ border-top: 1px solid var(--line, #dfe1d9);
+ border-bottom: 1px solid var(--line, #dfe1d9);
+ border-radius: 0;
}
.guide input:not([type="checkbox"]),
.guide textarea {
diff --git a/packages/workflows-ui/src/workflow-guide.tsx b/packages/workflows-ui/src/workflow-guide.tsx
index ab757a3d2..d6ec4abf8 100644
--- a/packages/workflows-ui/src/workflow-guide.tsx
+++ b/packages/workflows-ui/src/workflow-guide.tsx
@@ -19,7 +19,12 @@ import {
} from "lucide-react";
import type { WorkflowMap } from "./model";
import type { WorkflowsPlatform } from "./platform";
-import { guideHtml, guideImage, type WorkflowGuide as Guide } from "./guide";
+import {
+ guideHtml,
+ guideImage,
+ guideSourceImage,
+ type WorkflowGuide as Guide,
+} from "./guide";
import { GuideAssistant } from "./guide-assistant";
import styles from "./workflow-guide.module.css";
@@ -379,8 +384,11 @@ export function WorkflowGuide({
{draft.steps.map((step, i) => {
const image =
!stale && step.includeImage
- ? guideImage(workflow, step.sourceStage)
+ ? guideImage(workflow, step.sourceStage, step.imageReview)
: null;
+ const source = !stale
+ ? guideSourceImage(workflow, step.sourceStage)
+ : null;
return (
<section
className={styles.step}
@@ -491,27 +499,35 @@ export function WorkflowGuide({
</button>
)}
</figure>
+ ) : source ? (
+ <ScreenshotReview
+ key={`${step.sourceStage}:${source.dataUrl}`}
+ source={source}
+ title={step.title}
+ include={() => {
+ update({
+ ...draft,
+ steps: draft.steps.map((s, j) =>
+ j === i
+ ? {
+ ...s,
+ includeImage: true,
+ imageReview: {
+ frameId: source.frameId,
+ timestamp: source.timestamp,
+ },
+ }
+ : s,
+ ),
+ });
+ }}
+ />
) : (
- <div className={styles.noImage}>
- <ImageOff size={15} />
- <span>No screenshot included</span>
- {editing &&
- !stale &&
- guideImage(workflow, step.sourceStage) && (
- <button
- onClick={() =>
- update({
- ...draft,
- steps: draft.steps.map((s, j) =>
- j === i ? { ...s, includeImage: true } : s,
- ),
- })
- }
- >
- Include source screenshot
- </button>
- )}
- </div>
+ <p className={styles.muted}>
+ {stale
+ ? "Source changed. Regenerate this SOP to review its screenshots."
+ : "No captured screenshot for this step."}
+ </p>
)}
{editing ? (
<label className={styles.result}>
@@ -638,3 +654,54 @@ export function WorkflowGuide({
</div>
);
}
+
+function ScreenshotReview({
+ source,
+ title,
+ include,
+}: {
+ source: NonNullable<ReturnType<typeof guideSourceImage>>;
+ title: string;
+ include: () => void;
+}) {
+ const [open, setOpen] = useState(false);
+ const [loaded, setLoaded] = useState(false);
+ const [failed, setFailed] = useState(false);
+ return (
+ <div className={styles.imageReview}>
+ <div className={styles.imageReviewHeader}>
+ <span>Saved screenshot available</span>
+ <button aria-expanded={open} onClick={() => setOpen(!open)}>
+ {open ? "Hide screenshot" : "Review screenshot"}
+ </button>
+ </div>
+ {open && (
+ <>
+ <img
+ src={source.dataUrl}
+ alt={`Review source for ${title}`}
+ draggable={false}
+ onLoad={() => {
+ setLoaded(true);
+ setFailed(false);
+ }}
+ onError={() => {
+ setFailed(true);
+ setLoaded(false);
+ }}
+ />
+ <div className={styles.imageReviewHeader}>
+ <span>
+ {failed
+ ? "This screenshot could not be loaded."
+ : "Does this image show the step clearly?"}
+ </span>
+ <button disabled={!loaded || failed} onClick={include}>
+ Include screenshot
+ </button>
+ </div>
+ </>
+ )}
+ </div>
+ );
+}
diff --git a/packages/workflows-ui/src/workflows-app.tsx b/packages/workflows-ui/src/workflows-app.tsx
index 07e0751f7..73ac21bb5 100644
--- a/packages/workflows-ui/src/workflows-app.tsx
+++ b/packages/workflows-ui/src/workflows-app.tsx
@@ -957,7 +957,7 @@ function WorkflowDetail({ workflow, navigate, platform, workProfile, saveCorrect
</li>)}</ol> : <p>Step not yet verified.</p>}
{!!stage.openQuestions?.length && <details className={styles.procedureQuestions}><summary>Unresolved details</summary><ul>{stage.openQuestions.map((question) => <li key={question}>{question}</li>)}</ul></details>}
</section>
- {stage.screenshot && <div className={styles.stageScreenshot}>
+ {stage.screenshot?.visualVerified && <div className={styles.stageScreenshot}>
<>
<div className={styles.screenshotFrame}><img src={stage.screenshot.dataUrl} alt={`Captured screen evidence for ${stage.name}`} draggable={false} data-lm-disable="true" /></div>
<div><Camera size={12} /><span>{formatEvidenceTimestamp(stage.screenshot.timestamp)} · {stage.screenshot.app} · {screenshotMatchLabel(stage.screenshot.matchDistanceSeconds)}</span><CapturedMomentButton frameId={stage.screenshot.frameId} timestamp={stage.screenshot.timestamp} open={platform.openCapturedMoment} /></div>