YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(workflows): select images for moving and deletion without resizing
Louis Beaumont5dc79cdca0c4
diff --git a/apps/screenpipe-app-tauri/components/workflows/step-evidence.test.tsx b/apps/screenpipe-app-tauri/components/workflows/step-evidence.test.tsxindex dfca9e813..e5f2bbcd9 100644--- a/apps/screenpipe-app-tauri/components/workflows/step-evidence.test.tsx+++ b/apps/screenpipe-app-tauri/components/workflows/step-evidence.test.tsx@@ -105,16 +105,16 @@ describe("missing attached screenshots", () => {});describe("step evidence media", () => {- it("shows all unique verified images, keeps independent sizing and opens the selected frame", async () => {+ it("shows all unique images, selects without resizing and opens the selected frame", async () => {const second = { ...stage.screenshot!, frameId: 9002, timestamp: "2026-09-22T11:02:00Z" };const multi = { ...stage, screenshots: [stage.screenshot!, second, second, { ...second, frameId: 9003, visualVerified: false }] };const open = vi.fn().mockResolvedValue(undefined);render(<WorkflowStepEvidence workflow={workflow} stage={multi} platform={{ openCapturedMoment: open } as unknown as WorkflowsPlatform} />);expect(screen.getAllByRole("img")).toHaveLength(2);- expect(screen.getAllByRole("button", { name: /Reduce screenshot/ })).toHaveLength(2);- fireEvent.click(screen.getByRole("button", { name: `Reduce screenshot 1 for ${stage.name}` }));- expect(screen.getByRole("button", { name: `Enlarge screenshot 1 for ${stage.name}` })).toHaveAttribute("aria-expanded", "false");- expect(screen.getByRole("button", { name: `Reduce screenshot 2 for ${stage.name}` })).toHaveAttribute("aria-expanded", "true");+ expect(screen.getAllByRole("button", { name: /Select screenshot/ })).toHaveLength(2);+ fireEvent.click(screen.getByRole("button", { name: `Select screenshot 1 for ${stage.name}` }));+ expect(screen.getByRole("button", { name: `Select screenshot 1 for ${stage.name}` })).toHaveAttribute("aria-pressed", "true");+ expect(screen.getByRole("button", { name: `Select screenshot 2 for ${stage.name}` })).toHaveAttribute("aria-pressed", "false");fireEvent.click(screen.getByRole("button", { name: `Open recording 2 for ${stage.name}` }));await waitFor(() => expect(open).toHaveBeenCalledWith(second.frameId, second.timestamp));});@@ -132,7 +132,7 @@ describe("step evidence media", () => {it("shows the verified screenshot large by default without a redundant source list", () => {render(<WorkflowStepEvidence workflow={workflow} stage={stage} platform={{} as WorkflowsPlatform} />);expect(screen.getByRole("img", { name: `Captured reference for ${stage.name}` })).toBeVisible();- expect(screen.getByRole("button", { name: `Reduce screenshot for ${stage.name}` })).toHaveAttribute("aria-expanded", "true");+ expect(screen.getByRole("button", { name: `Select screenshot for ${stage.name}` })).toHaveAttribute("aria-pressed", "false");expect(screen.queryByRole("button", { name: /sources?$/ })).not.toBeInTheDocument();expect(screen.queryByRole("region", { name: `Sources for ${stage.name}` })).not.toBeInTheDocument();});@@ -243,3 +243,29 @@ describe("recorder references", () => {expect(await screen.findByRole("img")).toHaveAttribute("src", "blob:available");});});++it("selects, moves and deletes images through the authored layout without altering captures", () => {+ const second = { ...stage.screenshot!, frameId: 9002, timestamp: "2026-09-22T11:02:00Z" };+ const multi = { ...stage, screenshots: [stage.screenshot!, second] };+ const change = vi.fn();+ const { rerender } = render(<WorkflowStepEvidence workflow={workflow} stage={multi} platform={{} as WorkflowsPlatform} onImageLayoutChange={change} />);+ const select = screen.getByRole("button", { name: `Select screenshot 2 for ${stage.name}` });+ fireEvent.click(select);+ expect(change).not.toHaveBeenCalled();+ expect(screen.getByRole("toolbar", { name: "Selected image" })).toBeVisible();+ fireEvent.click(screen.getByRole("button", { name: "Move image up" }));+ let layout = change.mock.lastCall![0];+ expect(layout.order).toEqual([second.frameId, stage.screenshot!.frameId]);+ rerender(<WorkflowStepEvidence workflow={workflow} stage={{ ...multi, imageLayout: layout }} platform={{} as WorkflowsPlatform} onImageLayoutChange={change} />);+ fireEvent.click(screen.getByRole("button", { name: "Move image up" }));+ layout = change.mock.lastCall![0];+ expect(layout.beforeText).toBe(true);+ fireEvent.keyDown(screen.getByRole("button", { name: `Select screenshot 1 for ${stage.name}` }), { key: "Delete" });+ layout = change.mock.lastCall![0];+ expect(layout.hidden).toEqual([second.frameId]);+ rerender(<WorkflowStepEvidence workflow={workflow} stage={{ ...multi, imageLayout: layout }} platform={{} as WorkflowsPlatform} onImageLayoutChange={change} />);+ expect(screen.getAllByRole("img")).toHaveLength(1);+ expect(multi.screenshots).toHaveLength(2);+ fireEvent.click(screen.getByRole("button", { name: "Restore deleted images" }));+ expect(change.mock.lastCall![0].hidden).toEqual([]);+});diff --git a/crates/screenpipe-engine/src/routes/workflow_edits.rs b/crates/screenpipe-engine/src/routes/workflow_edits.rsindex c82ade86d..3f1eca991 100644--- a/crates/screenpipe-engine/src/routes/workflow_edits.rs+++ b/crates/screenpipe-engine/src/routes/workflow_edits.rs@@ -10,7 +10,7 @@ use super::{};use axum::{http::StatusCode, Extension, Json};use oasgen::{oasgen, OaSchema};-use serde::Deserialize;+use serde::{Deserialize, Serialize};use serde_json::{json, Value};use std::collections::HashSet;@@ -29,11 +29,33 @@ pub struct EditRequest {#[derive(Deserialize, OaSchema)]#[serde(deny_unknown_fields, rename_all = "camelCase")]pub struct StageEdit {+ pub image_layout: Option<ImageLayout>,pub source_index: Option<usize>,pub name: String,pub description: String,pub procedure: Vec<ProcedureEdit>,}+#[derive(Deserialize, Serialize, OaSchema)]+#[serde(deny_unknown_fields, rename_all = "camelCase")]+pub struct ImageLayout {+ pub order: Vec<u64>,+ pub hidden: Vec<u64>,+ pub before_text: bool,+}+fn image_layout_ok(layout: &ImageLayout) -> bool {+ [&layout.order, &layout.hidden].iter().all(|ids| {+ ids.len() <= 100+ && ids.iter().all(|id| *id <= 9_007_199_254_740_991)+ && ids.iter().collect::<HashSet<_>>().len() == ids.len()+ })+}+fn without_image_layout(value: &Value) -> Value {+ let mut value = value.clone();+ if let Some(object) = value.as_object_mut() {+ object.remove("imageLayout");+ }+ value+}#[derive(Deserialize, OaSchema)]#[serde(deny_unknown_fields, rename_all = "camelCase")]pub struct ProcedureEdit {@@ -73,7 +95,13 @@ pub(super) fn preserve_edits(previous: &Value, next: &mut Value) {.filter(|candidate| candidate["name"] == stage["name"]).collect();if stage["userEdited"] != true && matches.len() == 1 {- matches[0].clone()+ let mut enriched = matches[0].clone();+ if let Some(layout) = stage.get("imageLayout") {+ enriched["imageLayout"] = layout.clone();+ } else if let Some(object) = enriched.as_object_mut() {+ object.remove("imageLayout");+ }+ enriched} else {stage.clone()}@@ -81,7 +109,17 @@ pub(super) fn preserve_edits(previous: &Value, next: &mut Value) {.collect::<Vec<_>>());// Prior edits alone do not invalidate newly verified timing. Only// invalidate when preserving owner changes alters the reviewed map.- if next["stages"] != json!(proposed) {+ if next["stages"]+ .as_array()+ .unwrap()+ .iter()+ .map(without_image_layout)+ .collect::<Vec<_>>()+ != proposed+ .iter()+ .map(without_image_layout)+ .collect::<Vec<_>>()+ {invalidate_step_claims(next);}}@@ -115,6 +153,10 @@ fn apply_edit(previous: &Value, body: &EditRequest) -> Result<Value, ApiError> {if !text_ok(&stage.name, true)|| !text_ok(&stage.description, false)|| stage.procedure.len() > 100+ || stage+ .image_layout+ .as_ref()+ .is_some_and(|layout| !image_layout_ok(layout)){return Err(invalid());}@@ -163,6 +205,9 @@ fn apply_edit(previous: &Value, body: &EditRequest) -> Result<Value, ApiError> {next["timing"] = Value::Null;next["userEdited"] = json!(true);}+ if let Some(layout) = &stage.image_layout {+ next["imageLayout"] = json!(layout);+ }stages.push(next);}let mut next = previous.clone();@@ -183,22 +228,29 @@ fn apply_edit(previous: &Value, body: &EditRequest) -> Result<Value, ApiError> {}}if stages != *original {+ let presentation_only = stages.iter().map(without_image_layout).collect::<Vec<_>>()+ == original+ .iter()+ .map(without_image_layout)+ .collect::<Vec<_>>();next["stages"] = json!(stages);edits.insert("stages".into(), json!(true));- // Rename surviving step associations and remove associations to deleted steps.- let mut friction = vec![];- for item in previous["bottlenecks"].as_array().into_iter().flatten() {- if let Some(name) =- names.get(&item["stage"].as_str().unwrap_or_default().to_lowercase())- {- let mut item = item.clone();- item["stage"] = json!(name);- friction.push(item);+ if !presentation_only {+ // Rename surviving step associations and remove associations to deleted steps.+ let mut friction = vec![];+ for item in previous["bottlenecks"].as_array().into_iter().flatten() {+ if let Some(name) =+ names.get(&item["stage"].as_str().unwrap_or_default().to_lowercase())+ {+ let mut item = item.clone();+ item["stage"] = json!(name);+ friction.push(item);+ }}+ next["bottlenecks"] = json!(friction);+ edits.insert("bottlenecks".into(), json!(true));+ invalidate_step_claims(&mut next);}- next["bottlenecks"] = json!(friction);- edits.insert("bottlenecks".into(), json!(true));- invalidate_step_claims(&mut next);}if next == *previous {return Ok(next);@@ -253,6 +305,39 @@ mod tests {{"sourceIndex":1,"name":"Write","description":"Draft report","procedure":[]}]})).unwrap()}#[test]+ fn image_selection_edits_preserve_sources_timing_and_survive_discovery() {+ let prior = original();+ let mut body = request();+ body.stages[0].image_layout = Some(ImageLayout {+ order: vec![2, 1],+ hidden: vec![1],+ before_text: true,+ });+ let saved = apply_edit(&prior, &body).unwrap();+ assert_eq!(+ saved["stages"][0]["screenshots"],+ prior["stages"][0]["screenshots"]+ );+ assert_eq!(+ saved["stages"][0]["evidence"],+ prior["stages"][0]["evidence"]+ );+ assert_eq!(saved["timing"], prior["timing"]);+ assert_eq!(saved["stages"][0]["imageLayout"]["hidden"], json!([1]));+ let mut next = prior.clone();+ preserve_edits(&saved, &mut next);+ assert_eq!(+ next["stages"][0]["imageLayout"],+ saved["stages"][0]["imageLayout"]+ );+ assert_eq!(next["timing"], prior["timing"]);+ body.stages[0].image_layout.as_mut().unwrap().order = vec![1, 1];+ assert_eq!(+ apply_edit(&prior, &body).unwrap_err().0,+ StatusCode::BAD_REQUEST+ );+ }+ #[test]fn unchanged_save_is_noop_and_conflict_does_not_mutate() {let prior = original();let mut body = request();@@ -318,6 +403,7 @@ mod tests {assert_eq!(saved["revision"], 4);body.stages.remove(1);body.stages.push(StageEdit {+ image_layout: None,source_index: None,name: "Deliver".into(),description: "".into(),diff --git a/docs/COVERAGE.md b/docs/COVERAGE.mdindex 791584e78..70edb30d3 100644--- a/docs/COVERAGE.md+++ b/docs/COVERAGE.md@@ -38,15 +38,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.- Mapped suites: 37- Mapped Rust files: 396-- Active test blocks: 3874+- Active test blocks: 3875- Ignored/manual test blocks: 181-- Weighted coverage points: 3198.5+- Weighted coverage points: 3199.2| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 33 | 3726 | 166 | 3131.5 | 21 | 11 | 100% |-| macos | 32 | 3754 | 142 | 3120.8 | 22 | 11 | 100% |-| linux | 29 | 3304 | 141 | 2749.8 | 20 | 11 | 100% |+| windows | 33 | 3727 | 166 | 3132.2 | 21 | 11 | 100% |+| macos | 32 | 3755 | 142 | 3121.5 | 22 | 11 | 100% |+| linux | 29 | 3305 | 141 | 2750.5 | 20 | 11 | 100% |## Refreshdiff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex 21fde83f2..140ca8071 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -9,10 +9,10 @@ confidence, and criticality.- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs- Mapped suites: 37- Mapped Rust files: 396-- Active test blocks: 3874+- Active test blocks: 3875- Ignored/manual test blocks: 181-- Declared test blocks: 4055-- Weighted coverage points: 3198.5+- Declared test blocks: 4056+- Weighted coverage points: 3199.2Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.Criticality weights: high=1.0, medium=0.7, low=0.4.@@ -23,15 +23,15 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |-| windows | 33 | 3726 | 166 | 3131.5 | 21 | 11 | 100% |-| macos | 32 | 3754 | 142 | 3120.8 | 22 | 11 | 100% |-| linux | 29 | 3304 | 141 | 2749.8 | 20 | 11 | 100% |+| windows | 33 | 3727 | 166 | 3132.2 | 21 | 11 | 100% |+| macos | 32 | 3755 | 142 | 3121.5 | 22 | 11 | 100% |+| linux | 29 | 3305 | 141 | 2750.5 | 20 | 11 | 100% |## Crate Summary| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- |-| screenpipe-engine | 12 | 22 | 131 | 1858 | 53 | 1431.3 | 10 |+| screenpipe-engine | 12 | 22 | 131 | 1859 | 53 | 1432.0 | 10 || screenpipe-db | 5 | 68 | 28 | 629 | 31 | 599.4 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 || screenpipe-audio | 8 | 29 | 53 | 695 | 53 | 606.9 | 5 |@@ -69,7 +69,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| database | 7 suites / 585 active / 27 ignored / 555.4 pts | 7 suites / 585 active / 27 ignored / 555.4 pts | 7 suites / 585 active / 27 ignored / 555.4 pts || db-search | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts || engine-lifecycle | 6 suites / 270 active / 1 ignored / 242.0 pts | 6 suites / 270 active / 1 ignored / 242.0 pts | 6 suites / 266 active / 9 ignored / 241.1 pts |-| local-api | 3 suites / 481 active / 12 ignored / 340.6 pts | 3 suites / 481 active / 12 ignored / 340.6 pts | 3 suites / 481 active / 12 ignored / 340.6 pts |+| local-api | 3 suites / 482 active / 12 ignored / 341.3 pts | 3 suites / 482 active / 12 ignored / 341.3 pts | 3 suites / 482 active / 12 ignored / 341.3 pts || meeting | 6 suites / 1753 active / 32 ignored / 1434.4 pts | 6 suites / 1753 active / 32 ignored / 1434.4 pts | 4 suites / 1399 active / 28 ignored / 1104.1 pts || ocr | 4 suites / 141 active / 7 ignored / 130.8 pts | 4 suites / 134 active / 7 ignored / 128.6 pts | 3 suites / 125 active / 6 ignored / 119.6 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |@@ -78,7 +78,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| privacy | 5 suites / 1013 active / 42 ignored / 816.8 pts | 5 suites / 1054 active / 18 ignored / 812.5 pts | 5 suites / 971 active / 14 ignored / 781.7 pts || real-app | - | 1 suites / 110 active / 5 ignored / 44.0 pts | - || speaker | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts |-| storage | 4 suites / 689 active / 41 ignored / 574.7 pts | 4 suites / 689 active / 41 ignored / 574.7 pts | 4 suites / 689 active / 41 ignored / 574.7 pts |+| storage | 4 suites / 690 active / 41 ignored / 575.4 pts | 4 suites / 690 active / 41 ignored / 575.4 pts | 4 suites / 690 active / 41 ignored / 575.4 pts || sync | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts || timeline | 4 suites / 1239 active / 48 ignored / 1017.6 pts | 4 suites / 1239 active / 48 ignored / 1017.6 pts | 4 suites / 1239 active / 48 ignored / 1017.6 pts || transcription | 7 suites / 892 active / 53 ignored / 698.0 pts | 6 suites / 886 active / 53 ignored / 693.8 pts | 6 suites / 886 active / 53 ignored / 693.8 pts |@@ -147,7 +147,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| engine-meeting-watcher | screenpipe-engine | windows, macos | meeting | meeting-live-notes | high | strong | mixed | 10 | 275 | 3 | Successor of src/meeting_detector.rs and src/meeting_telemetry.rs. Audio-process and UI-scan backend state machines, candidate resolution, shared telemetry, and a scored trajectory eval. ui_scan/macos.rs and ui_scan/windows.rs are cfg-gated and only execute on their target OS; both backends are null on Linux. || engine-retention-storage | screenpipe-engine | windows, macos, linux | storage, engine-lifecycle, performance | engine-health-lifecycle, performance-liveness | medium | strong | mixed | 5 | 39 | 0 | Local retention deletion (including lean-mode heavy-text stripping), cloud archive watermarking, atomic state-file replacement, and the low-disk capture monitor. || engine-telemetry-observability | screenpipe-engine | windows, macos, linux | engine-lifecycle, performance | engine-health-lifecycle, performance-liveness | medium | strong | unit | 8 | 40 | 0 | PostHog capture gating, telemetry context shaping, piggyback telemetry forwarding, crash-log helpers, resource monitoring, and the recording-coverage reliability metric. |-| engine-workflows | screenpipe-engine | windows, macos, linux | local-api, storage | local-api-search | high | partial | mixed | 7 | 36 | 0 | Catalog identity and persistence, revision-scoped feedback, workflow evidence and pipeline contracts, and denial of agent-granted rollout access. Does not assert live cloud discovery quality. Workflow publication orchestration deadlines and nested query backpressure are covered with virtual-time router tests. |+| engine-workflows | screenpipe-engine | windows, macos, linux | local-api, storage | local-api-search | high | partial | mixed | 7 | 37 | 0 | Catalog identity and persistence, revision-scoped feedback, workflow evidence and pipeline contracts, and denial of agent-granted rollout access. Does not assert live cloud discovery quality. Workflow publication orchestration deadlines and nested query backpressure are covered with virtual-time router tests. || filesystem-durable-sync | screenpipe-fs | macos, linux, windows | database, vision-capture, audio | capture-ocr-pipeline, audio-record-transcribe, engine-health-lifecycle | high | strong | unit | 1 | 3 | 0 | Durable flushes preserve local full-sync behavior, fall back to fsync only for macOS ENOTSUP, and propagate real I/O or fallback failures. Real SMB migration and continued recording are covered by the marked-volume integration test. || screen-capture-ocr-contract | screenpipe-screen | windows, macos, linux | vision-capture, ocr | capture-ocr-pipeline | high | partial | unit | 3 | 18 | 0 | Cross-platform cached-OCR unit coverage for RawCaptureResult to CaptureResult metadata, browser URL, focus state, window-to-screen OCR coordinate transformation, Tesseract output parsing, and contour-based text-region detection for the meeting OCR gate. || screen-capture-windowing | screenpipe-screen | windows, macos, linux | vision-capture, timeline, performance, privacy | capture-ocr-pipeline, timeline-streaming, privacy-and-redaction, performance-liveness | high | strong | mixed | 14 | 193 | 0 | Window filtering, empty-window regressions, retry policy, URL timing, monitor cache, OCR cache, snapshots, and image comparison. |@@ -482,7 +482,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| engine-api-routes | screenpipe-engine | src/routes/timezone.rs | source | 8 | 0 | 8 || engine-api-routes | screenpipe-engine | src/routes/websocket.rs | source | 4 | 0 | 4 || engine-workflows | screenpipe-engine | src/routes/workflow_catalog.rs | source | 7 | 0 | 7 |-| engine-workflows | screenpipe-engine | src/routes/workflow_edits.rs | source | 12 | 0 | 12 |+| engine-workflows | screenpipe-engine | src/routes/workflow_edits.rs | source | 13 | 0 | 13 || engine-workflows | screenpipe-engine | src/routes/workflow_pipeline.rs | source | 4 | 0 | 4 || engine-workflows | screenpipe-engine | src/routes/workflows.rs | source | 3 | 0 | 3 || engine-config-lifecycle | screenpipe-engine | src/schedule_monitor.rs | source | 8 | 0 | 8 |diff --git a/docs/pr-assets/sop-share-images/workflow-selected.png b/docs/pr-assets/sop-share-images/workflow-selected.pngnew file mode 100644index 000000000..ee2a53724Binary files /dev/null and b/docs/pr-assets/sop-share-images/workflow-selected.png differdiff --git a/packages/workflows-ui/src/fixture-platform.ts b/packages/workflows-ui/src/fixture-platform.tsindex 4db45f0d9..3f61333d0 100644--- a/packages/workflows-ui/src/fixture-platform.ts+++ b/packages/workflows-ui/src/fixture-platform.ts@@ -541,7 +541,8 @@ export function createFixtureWorkflowsPlatform(analysis: WorkflowAnalysis = fixtconst prior = s.sourceIndex === null ? null : workflow.stages[s.sourceIndex];return { ...(prior ?? { activeMinutes: 0, waitingMinutes: 0, apps: [], confidence: 0, observedOccurrences: 0, observedDays: 0, evidence: [] }),timing: isUnchangedStage(s, prior ?? undefined) ? prior?.timing : null,- name: s.name.trim(), description: s.description.trim(), userEdited: true,+ name: s.name.trim(), description: s.description.trim(), userEdited: prior?.userEdited || !isUnchangedStage(s, prior ?? undefined),+ imageLayout: s.imageLayout,procedure: s.procedure.map(p => ({ ...(p.sourceIndex === null ? { quote: "", timestamp: "", app: "" } : prior?.procedure?.[p.sourceIndex] ?? { quote: "", timestamp: "", app: "" }), kind: p.kind, text: p.text.trim(), userEdited: true })),};});diff --git a/packages/workflows-ui/src/model.ts b/packages/workflows-ui/src/model.tsindex b896c315a..fe4b97343 100644--- a/packages/workflows-ui/src/model.ts+++ b/packages/workflows-ui/src/model.ts@@ -92,7 +92,10 @@ export type WorkflowScreenshot = {dataUrl: string;};+export type WorkflowImageLayout = { order: number[]; hidden: number[]; beforeText: boolean };+export type WorkflowStage = {+ imageLayout?: WorkflowImageLayout;timing?: WorkflowTiming | null;userEdited?: boolean;name: string;diff --git a/packages/workflows-ui/src/workflow-editor.tsx b/packages/workflows-ui/src/workflow-editor.tsxindex dc6fff0fd..9942c2ef3 100644--- a/packages/workflows-ui/src/workflow-editor.tsx+++ b/packages/workflows-ui/src/workflow-editor.tsx@@ -61,7 +61,7 @@ export function WorkflowEditor({ workflow, save, actions, renderSource, stepsOnlworkflow: WorkflowMap;save: (draft: WorkflowEdit) => Promise<WorkflowMap>;actions?: ReactNode;- renderSource?: (stage: WorkflowStage) => ReactNode;+ renderSource?: (stage: WorkflowStage, onImageLayoutChange?: (layout: NonNullable<WorkflowStage["imageLayout"]>) => void) => ReactNode;}) {const key = `screenpipe:workflow-edit:${workflow.id ?? workflow.title}`;const [initial, setInitial] = useState(() => keyedDraft(workflowEdit(workflow)));@@ -415,6 +415,7 @@ export function WorkflowEditor({ workflow, save, actions, renderSource, stepsOnl? workflow.stages[stage.sourceIndex]?.timing : null} /></div></div>+ <div style={{ display: "flex", flexDirection: "column" }}><Textrichlabel={`Step ${index + 1} description`}@@ -554,7 +555,8 @@ export function WorkflowEditor({ workflow, save, actions, renderSource, stepsOnldisabled={stage.procedure.length >= 100} onClick={() => addDetail(index)}><Plus size={14} aria-hidden="true" />{singleCheck ? "Add expected result" : "Add detail"}</button>}- {stage.sourceIndex !== null && workflow.stages[stage.sourceIndex] && <div className={styles.source}>{renderSource?.(workflow.stages[stage.sourceIndex])}</div>}+ {stage.sourceIndex !== null && workflow.stages[stage.sourceIndex] && <div className={styles.source} style={stage.imageLayout?.beforeText ? { order: -1 } : undefined}>{renderSource?.({ ...workflow.stages[stage.sourceIndex], imageLayout: stage.imageLayout }, layout => stageChange(index, { ...stage, imageLayout: layout }))}</div>}+ </div></article></Fragment>))}@@ -591,7 +593,7 @@ export function WorkflowDocument({ workflow, actions, sourceLabel, renderSourceworkflow: WorkflowMap;actions?: ReactNode;sourceLabel?: string;- renderSource?: (stage: WorkflowStage) => ReactNode;+ renderSource?: (stage: WorkflowStage, onImageLayoutChange?: (layout: NonNullable<WorkflowStage["imageLayout"]>) => void) => ReactNode;}) {return <section className={styles.editor} aria-label="Workflow document"><header className={styles.toolbar}>@@ -615,6 +617,7 @@ export function WorkflowDocument({ workflow, actions, sourceLabel, renderSource<h2 className={`${styles.title} ${styles.readOnlyText}`}>{stage.name}</h2><div className={styles.stepActions}><TimingDisclosure hideWhenUnknown showDuration label={`Timing for step ${index + 1}`} value={stage.timing} /></div></div>+ <div style={{ display: "flex", flexDirection: "column" }}>{stage.description && <WorkflowRichText label={`Step ${index + 1} description`} value={stage.description} />}<div className={styles.blocks}>{stage.procedure?.map((detail, detailIndex) => {@@ -635,7 +638,8 @@ export function WorkflowDocument({ workflow, actions, sourceLabel, renderSource})}</div>{!!stage.apps.length && <p className={styles.stepApps}>{stage.apps.join(" · ")}</p>}- {renderSource && <div className={styles.source}>{renderSource(stage)}</div>}+ {renderSource && <div className={styles.source} style={stage.imageLayout?.beforeText ? { order: -1 } : undefined}>{renderSource(stage)}</div>}+ </div></article>)}</div><div className={styles.endpoint}>Ends with<span className={styles.readOnlyText}>{workflow.outcome}</span></div>diff --git a/packages/workflows-ui/src/workflow-edits.ts b/packages/workflows-ui/src/workflow-edits.tsindex cf654616c..b119adb47 100644--- a/packages/workflows-ui/src/workflow-edits.ts+++ b/packages/workflows-ui/src/workflow-edits.ts@@ -9,6 +9,7 @@ export type ProcedureEdit = {};export type StageEdit = {sourceIndex: number | null;+ imageLayout?: WorkflowStage["imageLayout"];name: string;description: string;procedure: ProcedureEdit[];@@ -42,6 +43,7 @@ export function workflowEdit(workflow: WorkflowMap): WorkflowEdit {sourceIndex,name: stage.name,description: stage.description,+ ...(stage.imageLayout ? { imageLayout: stage.imageLayout } : {}),procedure: (stage.procedure ?? []).map(({ kind, text }, sourceIndex) => ({sourceIndex,kind,@@ -86,6 +88,7 @@ export function validWorkflowEdit((s) =>text(s.name, true) &&text(s.description) &&+ (!s.imageLayout || (typeof s.imageLayout.beforeText === "boolean" && [s.imageLayout.order, s.imageLayout.hidden].every(ids => Array.isArray(ids) && ids.length <= 100 && new Set(ids).size === ids.length && ids.every(id => Number.isSafeInteger(id) && id >= 0)))) &&Array.isArray(s.procedure) &&s.procedure.length <= 100 &&s.procedure.every(diff --git a/packages/workflows-ui/src/workflow-step-evidence.module.css b/packages/workflows-ui/src/workflow-step-evidence.module.cssindex 9069dc88d..472eedfa2 100644--- a/packages/workflows-ui/src/workflow-step-evidence.module.css+++ b/packages/workflows-ui/src/workflow-step-evidence.module.css@@ -3,13 +3,9 @@.evidence { margin: 12px 0 4px; min-width: 0; }.previewStatus { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 16px; padding: 16px; margin-bottom: 8px; border: 1px solid var(--line); border-radius: 6px; color: var(--muted); font-size: 12px; }.previewStatus button { text-decoration: underline; }-.figure { max-width: 360px; margin: 0 0 20px; }-.figure.expanded { max-width: 100%; }-.figure .imageButton { display: block; padding: 0; position: relative; width: 100%; border: 1px solid var(--line); border-radius: 6px; overflow: hidden; background: var(--soft); cursor: zoom-in; }-.expanded .imageButton { cursor: zoom-out; }+.figure { position: relative; max-width: 100%; margin: 0 0 20px; }+.figure .imageButton { display: block; padding: 0; position: relative; width: 100%; border: 1px solid var(--line); border-radius: 6px; overflow: hidden; background: var(--soft); cursor: pointer; }.imageButton img { display: block; width: 100%; height: auto; object-fit: contain; }-.zoom { position: absolute; right: 8px; bottom: 8px; display: grid; place-items: center; padding: 6px; background: var(--panel); border: 1px solid var(--line); border-radius: 4px; opacity: 0; }-.imageButton:hover .zoom, .imageButton:focus-visible .zoom { opacity: 1; }.figure figcaption { display: flex; justify-content: space-between; align-items: center; gap: 8px; font-size: 11px; line-height: 1.5; color: var(--muted); }.figure figcaption > span { min-width: 0; overflow-wrap: anywhere; }.links { display: flex; flex-wrap: wrap; align-items: flex-start; gap: 4px 12px; font-size: 11px; color: var(--muted); }@@ -21,7 +17,6 @@/* Reveal playback without competing with the step. */@media (prefers-reduced-motion: no-preference) {- .zoom { transition: opacity 150ms ease; }.player { animation: evidenceReveal 150ms ease-out; }}@keyframes evidenceReveal { from { opacity: 0; transform: translateY(-3px); } to { opacity: 1; transform: translateY(0); } }@@ -32,3 +27,9 @@.evidence .recording:focus-visible { outline: 1px solid var(--ink); outline-offset: 2px; }.evidence .recording:disabled { opacity: .5; }@media (pointer: coarse) { .evidence .recording { width: 44px; height: 44px; } }++.selected .imageButton { outline: 2px solid var(--ink); outline-offset: 2px; }+.imageActions { position: absolute; top: 8px; right: 8px; display: flex; flex-wrap: wrap; gap: 4px; padding: 4px; max-width: calc(100% - 16px); background: var(--panel); border: 1px solid var(--line); border-radius: 6px; box-shadow: 0 2px 8px #0002; }+.imageActions button { display: inline-flex; align-items: center; gap: 4px; min-height: 32px; padding: 4px 8px; font: inherit; font-size: 12px; }+.imageActions button:disabled { opacity: .4; }+.restore { font: inherit; font-size: 12px; text-decoration: underline; }diff --git a/packages/workflows-ui/src/workflow-step-evidence.tsx b/packages/workflows-ui/src/workflow-step-evidence.tsxindex 9bb237712..f4a0a19e4 100644--- a/packages/workflows-ui/src/workflow-step-evidence.tsx+++ b/packages/workflows-ui/src/workflow-step-evidence.tsx@@ -3,7 +3,7 @@"use client";import { useEffect, useMemo, useRef, useState } from "react";-import { Loader2, Maximize2, Minimize2, Play, X } from "lucide-react";+import { Loader2, ArrowUp, ArrowDown, Trash2, Play, X } from "lucide-react";import type { WorkflowMap, WorkflowStage, WorkflowScreenshot } from "./model";import type { WorkflowsPlatform } from "./platform";import { WorkflowReplay } from "./workflow-replay";@@ -14,10 +14,11 @@ import styles from "./workflow-step-evidence.module.css";const when = (value: string) => Number.isFinite(Date.parse(value))? new Date(value).toLocaleString(undefined, { month: "short", day: "numeric", hour: "numeric", minute: "2-digit" }) : "Date unavailable";-export function WorkflowStepEvidence({ workflow, stage, platform }: {+export function WorkflowStepEvidence({ workflow, stage, platform, onImageLayoutChange }: {workflow: WorkflowMap; stage: WorkflowStage; platform: WorkflowsPlatform;+ onImageLayoutChange?: (layout: NonNullable<WorkflowStage["imageLayout"]>) => void;}) {- const [reduced, setReduced] = useState<Set<number>>(() => new Set());+ const [selectedImage, setSelectedImage] = useState<number | null>(null);const [previewHeight, setPreviewHeight] = useState(0);const [selected, setSelected] = useState<WorkflowScreenshot | null>(null);const [playing, setPlaying] = useState(false);@@ -31,7 +32,27 @@ export function WorkflowStepEvidence({ workflow, stage, platform }: {} : stage] }), [workflow, stage, selected]);const attached = verifiedStageScreenshots(stage);const preview = useSourceScreenshot(stage, attached.length > 0, platform.loadWorkflowScreenshot);- const screenshots = attached.length ? attached : preview.image ? [preview.image] : [];+ const available = attached.length ? attached : preview.image ? [preview.image] : [];+ const layout = stage.imageLayout ?? { order: [], hidden: [], beforeText: false };+ const ordered = [...available].sort((a, b) => {+ const rank = (id: number) => { const index = layout.order.indexOf(id); return index < 0 ? layout.order.length : index; };+ return rank(a.frameId) - rank(b.frameId);+ });+ const screenshots = ordered.filter(image => !layout.hidden.includes(image.frameId));+ const removeImage = (id: number) => {+ onImageLayoutChange?.({ ...layout, hidden: [...new Set([...layout.hidden, id])] });+ setSelectedImage(null);+ };+ const moveImage = (index: number, direction: -1 | 1) => {+ const next = index + direction;+ if (next < 0 || next >= screenshots.length) {+ onImageLayoutChange?.({ ...layout, beforeText: direction === -1 });+ } else {+ const order = screenshots.map(image => image.frameId);+ [order[index], order[next]] = [order[next], order[index]];+ onImageLayoutChange?.({ ...layout, order });+ }+ };const screenshot = screenshots[0];const canReplay = platform.loadWorkflowRecording && stage.evidence.some(e => !["audio", "meeting"].includes(e.source ?? "") && Number.isFinite(Date.parse(e.timestamp)));const openRecording = async (capture = screenshot) => {@@ -54,19 +75,23 @@ export function WorkflowStepEvidence({ workflow, stage, platform }: {else if (media?.url && platform.releaseWorkflowRecording) void platform.releaseWorkflowRecording(media.url).catch(() => {});}};- return <section ref={preview.ref} onLoad={() => setPreviewHeight(preview.ref.current?.getBoundingClientRect().height ?? 0)} style={!attached.length && !screenshots.length && previewHeight ? { minHeight: previewHeight } : undefined} className={`${styles.evidence} ph-no-capture ph-mask`} aria-label={`References for ${stage.name}`}>- {!playing && !screenshots.length && <div className={styles.previewStatus} role="status">+ return <section ref={preview.ref} onKeyDown={event => {+ if (event.key === "Escape") { setSelectedImage(null); event.stopPropagation(); }+ }} onBlur={event => { if (event.relatedTarget && !event.currentTarget.contains(event.relatedTarget as Node)) setSelectedImage(null); }} onLoad={() => setPreviewHeight(preview.ref.current?.getBoundingClientRect().height ?? 0)} style={!attached.length && !screenshots.length && previewHeight ? { minHeight: previewHeight } : undefined} className={`${styles.evidence} ph-no-capture ph-mask`} aria-label={`References for ${stage.name}`}>+ {!playing && !available.length && <div className={styles.previewStatus} role="status">{preview.status === "loading" ? "Loading source screenshot…" : preview.status === "error" ? "Could not load the source screenshot." : "No screenshot available for this step."}{preview.canRetry && preview.status !== "loading" && <button type="button" onClick={preview.retry}>Retry screenshot</button>}</div>}{!playing && screenshots.map((capture, index) => {- const expanded = !reduced.has(capture.frameId);const suffix = screenshots.length > 1 ? ` ${index + 1}` : "";- return <EvidenceFigure key={`${capture.frameId}:${capture.timestamp}`} capture={capture} stage={stage} platform={platform} suffix={suffix} expanded={expanded} opening={opening}- toggle={() => setReduced(previous => { const next = new Set(previous); if (expanded) next.add(capture.frameId); else next.delete(capture.frameId); return next; })}+ return <EvidenceFigure key={`${capture.frameId}:${capture.timestamp}`} capture={capture} stage={stage} platform={platform} suffix={suffix} selected={selectedImage === capture.frameId} opening={opening}+ select={() => setSelectedImage(capture.frameId)}+ actions={onImageLayoutChange ? { remove: () => removeImage(capture.frameId), up: () => moveImage(index, -1), down: () => moveImage(index, 1),+ upDisabled: index === 0 && layout.beforeText, downDisabled: index === screenshots.length - 1 && !layout.beforeText } : undefined}replay={canReplay || !!platform.openCapturedMoment ? () => void openRecording(capture) : undefined}resolved={capture === preview.image} />;})}+ {!playing && onImageLayoutChange && layout.hidden.length > 0 && <button type="button" className={styles.restore} onClick={() => onImageLayoutChange({ ...layout, hidden: [] })}>Restore deleted images</button>}{playing && <div ref={playerRef} className={styles.player}><button type="button" className={styles.close} aria-label={`Close recording for ${stage.name}`} title="Close recording" onClick={() => setPlaying(false)}><X size={15} /></button><WorkflowReplay workflow={replay} loadRecording={platform.loadWorkflowRecording} releaseRecording={platform.releaseWorkflowRecording} openCapturedMoment={platform.openCapturedMoment} />@@ -78,9 +103,10 @@ export function WorkflowStepEvidence({ workflow, stage, platform }: {</section>;}-function EvidenceFigure({ capture, stage, platform, suffix, expanded, opening, toggle, replay, resolved }: {+function EvidenceFigure({ capture, stage, platform, suffix, selected, opening, select, actions, replay, resolved }: {capture: WorkflowScreenshot; stage: WorkflowStage; platform: WorkflowsPlatform; suffix: string;- expanded: boolean; opening: boolean; toggle: () => void; replay?: () => void; resolved: boolean;+ selected: boolean; opening: boolean; select: () => void; replay?: () => void; resolved: boolean;+ actions?: { remove: () => void; up: () => void; down: () => void; upDisabled: boolean; downDisabled: boolean };}) {const load = resolved ? undefined : platform.loadWorkflowScreenshot;const preview = useSourceScreenshot(stage, !load, load, capture);@@ -89,14 +115,19 @@ function EvidenceFigure({ capture, stage, platform, suffix, expanded, opening, tconst [failed, setFailed] = useState(false);const [ratio, setRatio] = useState("16 / 10");useEffect(() => setFailed(false), [url]);- return <figure ref={preview.ref} className={`${styles.figure} ${expanded ? styles.expanded : ""}`}>- {url && !failed ? <button className={styles.imageButton} type="button" aria-label={`${expanded ? "Reduce" : "Enlarge"} screenshot${suffix} for ${stage.name}`} aria-expanded={expanded} onClick={toggle}>+ return <figure ref={preview.ref} className={`${styles.figure} ${selected ? styles.selected : ""}`}>+ {url && !failed ? <button className={styles.imageButton} type="button" aria-label={`Select screenshot${suffix} for ${stage.name}`} aria-pressed={selected} onClick={select}+ onKeyDown={event => { if (selected && actions && ["Delete", "Backspace"].includes(event.key)) { event.preventDefault(); event.stopPropagation(); actions.remove(); } }}><img src={url} alt={`Captured reference${suffix} for ${stage.name}`} loading="lazy" decoding="async" draggable={false} data-lm-disable="true" onLoad={event => { const image = event.currentTarget; if (image.naturalWidth && image.naturalHeight) setRatio(`${image.naturalWidth} / ${image.naturalHeight}`); }} onError={() => setFailed(true)} />- <span className={styles.zoom}>{expanded ? <Minimize2 size={15} /> : <Maximize2 size={15} />}</span></button> : <div className={styles.previewStatus} role="status" style={preview.status === "loading" && !failed ? { aspectRatio: ratio } : undefined}>{failed || preview.status === "error" ? "Could not load the source screenshot." : preview.status === "loading" ? "Loading source screenshot…" : "This screenshot is no longer available."}{(failed || preview.status === "error") && load && <button onClick={() => { setFailed(false); preview.retry(); }}>Retry screenshot</button>}</div>}+ {selected && actions && <div className={styles.imageActions} role="toolbar" aria-label="Selected image">+ <button type="button" onClick={actions.up} disabled={actions.upDisabled} aria-label="Move image up"><ArrowUp size={15} />Move up</button>+ <button type="button" onClick={actions.down} disabled={actions.downDisabled} aria-label="Move image down"><ArrowDown size={15} />Move down</button>+ <button type="button" onClick={actions.remove} aria-label="Delete image"><Trash2 size={15} />Delete</button>+ </div>}<figcaption><span>{!capture.visualVerified ? "Source screenshot · " : ""}{capture.app} · {when(capture.timestamp)}</span>{replay && <button className={styles.recording} title="Open recording" type="button" disabled={opening} onClick={replay} aria-label={`${platform.openCapturedMoment ? "Open" : "View"} recording${suffix} for ${stage.name}`}>{opening ? <Loader2 size={15} aria-hidden="true" /> : <Play size={15} aria-hidden="true" />}</button>}</figcaption>diff --git a/packages/workflows-ui/src/workflows-app.tsx b/packages/workflows-ui/src/workflows-app.tsxindex dab644a88..22f3f8e97 100644--- a/packages/workflows-ui/src/workflows-app.tsx+++ b/packages/workflows-ui/src/workflows-app.tsx@@ -728,7 +728,7 @@ export function WorkflowDetail({ editorOptions, sourceLabel, observationsAvailabreturn (<><button className={styles.backButton} onClick={() => navigate("workflows")}><ArrowLeft size={14} />All workflows</button>- {saveEdits ? <WorkflowEditor {...editorOptions} key={workflow.id || workflow.title} workflow={workflow} save={saveEdits} actions={workflowActions} renderSource={observationsAvailable ? stage => <WorkflowStepEvidence workflow={workflow} stage={stage} platform={platform} /> : undefined} /> : <WorkflowDocument workflow={workflow} actions={workflowActions} sourceLabel={sourceLabel}+ {saveEdits ? <WorkflowEditor {...editorOptions} key={workflow.id || workflow.title} workflow={workflow} save={saveEdits} actions={workflowActions} renderSource={observationsAvailable ? (stage, onImageLayoutChange) => <WorkflowStepEvidence workflow={workflow} stage={stage} platform={platform} onImageLayoutChange={onImageLayoutChange} /> : undefined} /> : <WorkflowDocument workflow={workflow} actions={workflowActions} sourceLabel={sourceLabel}renderSource={observationsAvailable ? stage => <WorkflowStepEvidence workflow={workflow} stage={stage} platform={platform} /> : undefined} />}<div className={styles.workflowNotes}><p className={styles.workflowReviewState}>{workflow.userEditedAt ? ui("Edited by you · sources kept as references") : workflow.evidenceStatus === "supported-steps" ? ui("Source-backed steps · not execution-tested") : ui("Needs review")}</p>