Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(workflows): select images for moving and deletion without resizing

Louis Beaumont5dc79cdca0c4
diff --git a/apps/screenpipe-app-tauri/components/workflows/step-evidence.test.tsx b/apps/screenpipe-app-tauri/components/workflows/step-evidence.test.tsx
index dfca9e813..e5f2bbcd9 100644
--- a/apps/screenpipe-app-tauri/components/workflows/step-evidence.test.tsx
+++ b/apps/screenpipe-app-tauri/components/workflows/step-evidence.test.tsx
@@ -105,16 +105,16 @@ describe("missing attached screenshots", () => {
});
describe("step evidence media", () => {
- it("shows all unique verified images, keeps independent sizing and opens the selected frame", async () => {
+ it("shows all unique images, selects without resizing and opens the selected frame", async () => {
const second = { ...stage.screenshot!, frameId: 9002, timestamp: "2026-09-22T11:02:00Z" };
const multi = { ...stage, screenshots: [stage.screenshot!, second, second, { ...second, frameId: 9003, visualVerified: false }] };
const open = vi.fn().mockResolvedValue(undefined);
render(<WorkflowStepEvidence workflow={workflow} stage={multi} platform={{ openCapturedMoment: open } as unknown as WorkflowsPlatform} />);
expect(screen.getAllByRole("img")).toHaveLength(2);
- expect(screen.getAllByRole("button", { name: /Reduce screenshot/ })).toHaveLength(2);
- fireEvent.click(screen.getByRole("button", { name: `Reduce screenshot 1 for ${stage.name}` }));
- expect(screen.getByRole("button", { name: `Enlarge screenshot 1 for ${stage.name}` })).toHaveAttribute("aria-expanded", "false");
- expect(screen.getByRole("button", { name: `Reduce screenshot 2 for ${stage.name}` })).toHaveAttribute("aria-expanded", "true");
+ expect(screen.getAllByRole("button", { name: /Select screenshot/ })).toHaveLength(2);
+ fireEvent.click(screen.getByRole("button", { name: `Select screenshot 1 for ${stage.name}` }));
+ expect(screen.getByRole("button", { name: `Select screenshot 1 for ${stage.name}` })).toHaveAttribute("aria-pressed", "true");
+ expect(screen.getByRole("button", { name: `Select screenshot 2 for ${stage.name}` })).toHaveAttribute("aria-pressed", "false");
fireEvent.click(screen.getByRole("button", { name: `Open recording 2 for ${stage.name}` }));
await waitFor(() => expect(open).toHaveBeenCalledWith(second.frameId, second.timestamp));
});
@@ -132,7 +132,7 @@ describe("step evidence media", () => {
it("shows the verified screenshot large by default without a redundant source list", () => {
render(<WorkflowStepEvidence workflow={workflow} stage={stage} platform={{} as WorkflowsPlatform} />);
expect(screen.getByRole("img", { name: `Captured reference for ${stage.name}` })).toBeVisible();
- expect(screen.getByRole("button", { name: `Reduce screenshot for ${stage.name}` })).toHaveAttribute("aria-expanded", "true");
+ expect(screen.getByRole("button", { name: `Select screenshot for ${stage.name}` })).toHaveAttribute("aria-pressed", "false");
expect(screen.queryByRole("button", { name: /sources?$/ })).not.toBeInTheDocument();
expect(screen.queryByRole("region", { name: `Sources for ${stage.name}` })).not.toBeInTheDocument();
});
@@ -243,3 +243,29 @@ describe("recorder references", () => {
expect(await screen.findByRole("img")).toHaveAttribute("src", "blob:available");
});
});
+
+it("selects, moves and deletes images through the authored layout without altering captures", () => {
+ const second = { ...stage.screenshot!, frameId: 9002, timestamp: "2026-09-22T11:02:00Z" };
+ const multi = { ...stage, screenshots: [stage.screenshot!, second] };
+ const change = vi.fn();
+ const { rerender } = render(<WorkflowStepEvidence workflow={workflow} stage={multi} platform={{} as WorkflowsPlatform} onImageLayoutChange={change} />);
+ const select = screen.getByRole("button", { name: `Select screenshot 2 for ${stage.name}` });
+ fireEvent.click(select);
+ expect(change).not.toHaveBeenCalled();
+ expect(screen.getByRole("toolbar", { name: "Selected image" })).toBeVisible();
+ fireEvent.click(screen.getByRole("button", { name: "Move image up" }));
+ let layout = change.mock.lastCall![0];
+ expect(layout.order).toEqual([second.frameId, stage.screenshot!.frameId]);
+ rerender(<WorkflowStepEvidence workflow={workflow} stage={{ ...multi, imageLayout: layout }} platform={{} as WorkflowsPlatform} onImageLayoutChange={change} />);
+ fireEvent.click(screen.getByRole("button", { name: "Move image up" }));
+ layout = change.mock.lastCall![0];
+ expect(layout.beforeText).toBe(true);
+ fireEvent.keyDown(screen.getByRole("button", { name: `Select screenshot 1 for ${stage.name}` }), { key: "Delete" });
+ layout = change.mock.lastCall![0];
+ expect(layout.hidden).toEqual([second.frameId]);
+ rerender(<WorkflowStepEvidence workflow={workflow} stage={{ ...multi, imageLayout: layout }} platform={{} as WorkflowsPlatform} onImageLayoutChange={change} />);
+ expect(screen.getAllByRole("img")).toHaveLength(1);
+ expect(multi.screenshots).toHaveLength(2);
+ fireEvent.click(screen.getByRole("button", { name: "Restore deleted images" }));
+ expect(change.mock.lastCall![0].hidden).toEqual([]);
+});
diff --git a/crates/screenpipe-engine/src/routes/workflow_edits.rs b/crates/screenpipe-engine/src/routes/workflow_edits.rs
index c82ade86d..3f1eca991 100644
--- a/crates/screenpipe-engine/src/routes/workflow_edits.rs
+++ b/crates/screenpipe-engine/src/routes/workflow_edits.rs
@@ -10,7 +10,7 @@ use super::{
};
use axum::{http::StatusCode, Extension, Json};
use oasgen::{oasgen, OaSchema};
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::collections::HashSet;
@@ -29,11 +29,33 @@ pub struct EditRequest {
#[derive(Deserialize, OaSchema)]
#[serde(deny_unknown_fields, rename_all = "camelCase")]
pub struct StageEdit {
+ pub image_layout: Option<ImageLayout>,
pub source_index: Option<usize>,
pub name: String,
pub description: String,
pub procedure: Vec<ProcedureEdit>,
}
+#[derive(Deserialize, Serialize, OaSchema)]
+#[serde(deny_unknown_fields, rename_all = "camelCase")]
+pub struct ImageLayout {
+ pub order: Vec<u64>,
+ pub hidden: Vec<u64>,
+ pub before_text: bool,
+}
+fn image_layout_ok(layout: &ImageLayout) -> bool {
+ [&layout.order, &layout.hidden].iter().all(|ids| {
+ ids.len() <= 100
+ && ids.iter().all(|id| *id <= 9_007_199_254_740_991)
+ && ids.iter().collect::<HashSet<_>>().len() == ids.len()
+ })
+}
+fn without_image_layout(value: &Value) -> Value {
+ let mut value = value.clone();
+ if let Some(object) = value.as_object_mut() {
+ object.remove("imageLayout");
+ }
+ value
+}
#[derive(Deserialize, OaSchema)]
#[serde(deny_unknown_fields, rename_all = "camelCase")]
pub struct ProcedureEdit {
@@ -73,7 +95,13 @@ pub(super) fn preserve_edits(previous: &Value, next: &mut Value) {
.filter(|candidate| candidate["name"] == stage["name"])
.collect();
if stage["userEdited"] != true && matches.len() == 1 {
- matches[0].clone()
+ let mut enriched = matches[0].clone();
+ if let Some(layout) = stage.get("imageLayout") {
+ enriched["imageLayout"] = layout.clone();
+ } else if let Some(object) = enriched.as_object_mut() {
+ object.remove("imageLayout");
+ }
+ enriched
} else {
stage.clone()
}
@@ -81,7 +109,17 @@ pub(super) fn preserve_edits(previous: &Value, next: &mut Value) {
.collect::<Vec<_>>());
// Prior edits alone do not invalidate newly verified timing. Only
// invalidate when preserving owner changes alters the reviewed map.
- if next["stages"] != json!(proposed) {
+ if next["stages"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(without_image_layout)
+ .collect::<Vec<_>>()
+ != proposed
+ .iter()
+ .map(without_image_layout)
+ .collect::<Vec<_>>()
+ {
invalidate_step_claims(next);
}
}
@@ -115,6 +153,10 @@ fn apply_edit(previous: &Value, body: &EditRequest) -> Result<Value, ApiError> {
if !text_ok(&stage.name, true)
|| !text_ok(&stage.description, false)
|| stage.procedure.len() > 100
+ || stage
+ .image_layout
+ .as_ref()
+ .is_some_and(|layout| !image_layout_ok(layout))
{
return Err(invalid());
}
@@ -163,6 +205,9 @@ fn apply_edit(previous: &Value, body: &EditRequest) -> Result<Value, ApiError> {
next["timing"] = Value::Null;
next["userEdited"] = json!(true);
}
+ if let Some(layout) = &stage.image_layout {
+ next["imageLayout"] = json!(layout);
+ }
stages.push(next);
}
let mut next = previous.clone();
@@ -183,22 +228,29 @@ fn apply_edit(previous: &Value, body: &EditRequest) -> Result<Value, ApiError> {
}
}
if stages != *original {
+ let presentation_only = stages.iter().map(without_image_layout).collect::<Vec<_>>()
+ == original
+ .iter()
+ .map(without_image_layout)
+ .collect::<Vec<_>>();
next["stages"] = json!(stages);
edits.insert("stages".into(), json!(true));
- // Rename surviving step associations and remove associations to deleted steps.
- let mut friction = vec![];
- for item in previous["bottlenecks"].as_array().into_iter().flatten() {
- if let Some(name) =
- names.get(&item["stage"].as_str().unwrap_or_default().to_lowercase())
- {
- let mut item = item.clone();
- item["stage"] = json!(name);
- friction.push(item);
+ if !presentation_only {
+ // Rename surviving step associations and remove associations to deleted steps.
+ let mut friction = vec![];
+ for item in previous["bottlenecks"].as_array().into_iter().flatten() {
+ if let Some(name) =
+ names.get(&item["stage"].as_str().unwrap_or_default().to_lowercase())
+ {
+ let mut item = item.clone();
+ item["stage"] = json!(name);
+ friction.push(item);
+ }
}
+ next["bottlenecks"] = json!(friction);
+ edits.insert("bottlenecks".into(), json!(true));
+ invalidate_step_claims(&mut next);
}
- next["bottlenecks"] = json!(friction);
- edits.insert("bottlenecks".into(), json!(true));
- invalidate_step_claims(&mut next);
}
if next == *previous {
return Ok(next);
@@ -253,6 +305,39 @@ mod tests {
{"sourceIndex":1,"name":"Write","description":"Draft report","procedure":[]}]})).unwrap()
}
#[test]
+ fn image_selection_edits_preserve_sources_timing_and_survive_discovery() {
+ let prior = original();
+ let mut body = request();
+ body.stages[0].image_layout = Some(ImageLayout {
+ order: vec![2, 1],
+ hidden: vec![1],
+ before_text: true,
+ });
+ let saved = apply_edit(&prior, &body).unwrap();
+ assert_eq!(
+ saved["stages"][0]["screenshots"],
+ prior["stages"][0]["screenshots"]
+ );
+ assert_eq!(
+ saved["stages"][0]["evidence"],
+ prior["stages"][0]["evidence"]
+ );
+ assert_eq!(saved["timing"], prior["timing"]);
+ assert_eq!(saved["stages"][0]["imageLayout"]["hidden"], json!([1]));
+ let mut next = prior.clone();
+ preserve_edits(&saved, &mut next);
+ assert_eq!(
+ next["stages"][0]["imageLayout"],
+ saved["stages"][0]["imageLayout"]
+ );
+ assert_eq!(next["timing"], prior["timing"]);
+ body.stages[0].image_layout.as_mut().unwrap().order = vec![1, 1];
+ assert_eq!(
+ apply_edit(&prior, &body).unwrap_err().0,
+ StatusCode::BAD_REQUEST
+ );
+ }
+ #[test]
fn unchanged_save_is_noop_and_conflict_does_not_mutate() {
let prior = original();
let mut body = request();
@@ -318,6 +403,7 @@ mod tests {
assert_eq!(saved["revision"], 4);
body.stages.remove(1);
body.stages.push(StageEdit {
+ image_layout: None,
source_index: None,
name: "Deliver".into(),
description: "".into(),
diff --git a/docs/COVERAGE.md b/docs/COVERAGE.md
index 791584e78..70edb30d3 100644
--- a/docs/COVERAGE.md
+++ b/docs/COVERAGE.md
@@ -38,15 +38,15 @@ results and `cargo llvm-cov` data on top when judging release confidence.
- Mapped suites: 37
- Mapped Rust files: 396
-- Active test blocks: 3874
+- Active test blocks: 3875
- Ignored/manual test blocks: 181
-- Weighted coverage points: 3198.5
+- Weighted coverage points: 3199.2
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 33 | 3726 | 166 | 3131.5 | 21 | 11 | 100% |
-| macos | 32 | 3754 | 142 | 3120.8 | 22 | 11 | 100% |
-| linux | 29 | 3304 | 141 | 2749.8 | 20 | 11 | 100% |
+| windows | 33 | 3727 | 166 | 3132.2 | 21 | 11 | 100% |
+| macos | 32 | 3755 | 142 | 3121.5 | 22 | 11 | 100% |
+| linux | 29 | 3305 | 141 | 2750.5 | 20 | 11 | 100% |
## Refresh
diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.md
index 21fde83f2..140ca8071 100644
--- a/docs/coverage/CORE.md
+++ b/docs/coverage/CORE.md
@@ -9,10 +9,10 @@ confidence, and criticality.
- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs
- Mapped suites: 37
- Mapped Rust files: 396
-- Active test blocks: 3874
+- Active test blocks: 3875
- Ignored/manual test blocks: 181
-- Declared test blocks: 4055
-- Weighted coverage points: 3198.5
+- Declared test blocks: 4056
+- Weighted coverage points: 3199.2
Confidence weights: strong=1.0, partial=0.7, conditional=0.4, smoke=0.3.
Criticality weights: high=1.0, medium=0.7, low=0.4.
@@ -23,15 +23,15 @@ are explicitly enabled in a runtime lane.
| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| windows | 33 | 3726 | 166 | 3131.5 | 21 | 11 | 100% |
-| macos | 32 | 3754 | 142 | 3120.8 | 22 | 11 | 100% |
-| linux | 29 | 3304 | 141 | 2749.8 | 20 | 11 | 100% |
+| windows | 33 | 3727 | 166 | 3132.2 | 21 | 11 | 100% |
+| macos | 32 | 3755 | 142 | 3121.5 | 22 | 11 | 100% |
+| linux | 29 | 3305 | 141 | 2750.5 | 20 | 11 | 100% |
## Crate Summary
| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows |
| --- | --- | --- | --- | --- | --- | --- | --- |
-| screenpipe-engine | 12 | 22 | 131 | 1858 | 53 | 1431.3 | 10 |
+| screenpipe-engine | 12 | 22 | 131 | 1859 | 53 | 1432.0 | 10 |
| screenpipe-db | 5 | 68 | 28 | 629 | 31 | 599.4 | 9 |
| screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 |
| screenpipe-audio | 8 | 29 | 53 | 695 | 53 | 606.9 | 5 |
@@ -69,7 +69,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| database | 7 suites / 585 active / 27 ignored / 555.4 pts | 7 suites / 585 active / 27 ignored / 555.4 pts | 7 suites / 585 active / 27 ignored / 555.4 pts |
| db-search | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts | 2 suites / 120 active / 9 ignored / 120.0 pts |
| engine-lifecycle | 6 suites / 270 active / 1 ignored / 242.0 pts | 6 suites / 270 active / 1 ignored / 242.0 pts | 6 suites / 266 active / 9 ignored / 241.1 pts |
-| local-api | 3 suites / 481 active / 12 ignored / 340.6 pts | 3 suites / 481 active / 12 ignored / 340.6 pts | 3 suites / 481 active / 12 ignored / 340.6 pts |
+| local-api | 3 suites / 482 active / 12 ignored / 341.3 pts | 3 suites / 482 active / 12 ignored / 341.3 pts | 3 suites / 482 active / 12 ignored / 341.3 pts |
| meeting | 6 suites / 1753 active / 32 ignored / 1434.4 pts | 6 suites / 1753 active / 32 ignored / 1434.4 pts | 4 suites / 1399 active / 28 ignored / 1104.1 pts |
| ocr | 4 suites / 141 active / 7 ignored / 130.8 pts | 4 suites / 134 active / 7 ignored / 128.6 pts | 3 suites / 125 active / 6 ignored / 119.6 pts |
| os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |
@@ -78,7 +78,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| privacy | 5 suites / 1013 active / 42 ignored / 816.8 pts | 5 suites / 1054 active / 18 ignored / 812.5 pts | 5 suites / 971 active / 14 ignored / 781.7 pts |
| real-app | - | 1 suites / 110 active / 5 ignored / 44.0 pts | - |
| speaker | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts | 2 suites / 416 active / 18 ignored / 416.0 pts |
-| storage | 4 suites / 689 active / 41 ignored / 574.7 pts | 4 suites / 689 active / 41 ignored / 574.7 pts | 4 suites / 689 active / 41 ignored / 574.7 pts |
+| storage | 4 suites / 690 active / 41 ignored / 575.4 pts | 4 suites / 690 active / 41 ignored / 575.4 pts | 4 suites / 690 active / 41 ignored / 575.4 pts |
| sync | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts | 1 suites / 551 active / 3 ignored / 385.7 pts |
| timeline | 4 suites / 1239 active / 48 ignored / 1017.6 pts | 4 suites / 1239 active / 48 ignored / 1017.6 pts | 4 suites / 1239 active / 48 ignored / 1017.6 pts |
| transcription | 7 suites / 892 active / 53 ignored / 698.0 pts | 6 suites / 886 active / 53 ignored / 693.8 pts | 6 suites / 886 active / 53 ignored / 693.8 pts |
@@ -147,7 +147,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| engine-meeting-watcher | screenpipe-engine | windows, macos | meeting | meeting-live-notes | high | strong | mixed | 10 | 275 | 3 | Successor of src/meeting_detector.rs and src/meeting_telemetry.rs. Audio-process and UI-scan backend state machines, candidate resolution, shared telemetry, and a scored trajectory eval. ui_scan/macos.rs and ui_scan/windows.rs are cfg-gated and only execute on their target OS; both backends are null on Linux. |
| engine-retention-storage | screenpipe-engine | windows, macos, linux | storage, engine-lifecycle, performance | engine-health-lifecycle, performance-liveness | medium | strong | mixed | 5 | 39 | 0 | Local retention deletion (including lean-mode heavy-text stripping), cloud archive watermarking, atomic state-file replacement, and the low-disk capture monitor. |
| engine-telemetry-observability | screenpipe-engine | windows, macos, linux | engine-lifecycle, performance | engine-health-lifecycle, performance-liveness | medium | strong | unit | 8 | 40 | 0 | PostHog capture gating, telemetry context shaping, piggyback telemetry forwarding, crash-log helpers, resource monitoring, and the recording-coverage reliability metric. |
-| engine-workflows | screenpipe-engine | windows, macos, linux | local-api, storage | local-api-search | high | partial | mixed | 7 | 36 | 0 | Catalog identity and persistence, revision-scoped feedback, workflow evidence and pipeline contracts, and denial of agent-granted rollout access. Does not assert live cloud discovery quality. Workflow publication orchestration deadlines and nested query backpressure are covered with virtual-time router tests. |
+| engine-workflows | screenpipe-engine | windows, macos, linux | local-api, storage | local-api-search | high | partial | mixed | 7 | 37 | 0 | Catalog identity and persistence, revision-scoped feedback, workflow evidence and pipeline contracts, and denial of agent-granted rollout access. Does not assert live cloud discovery quality. Workflow publication orchestration deadlines and nested query backpressure are covered with virtual-time router tests. |
| filesystem-durable-sync | screenpipe-fs | macos, linux, windows | database, vision-capture, audio | capture-ocr-pipeline, audio-record-transcribe, engine-health-lifecycle | high | strong | unit | 1 | 3 | 0 | Durable flushes preserve local full-sync behavior, fall back to fsync only for macOS ENOTSUP, and propagate real I/O or fallback failures. Real SMB migration and continued recording are covered by the marked-volume integration test. |
| screen-capture-ocr-contract | screenpipe-screen | windows, macos, linux | vision-capture, ocr | capture-ocr-pipeline | high | partial | unit | 3 | 18 | 0 | Cross-platform cached-OCR unit coverage for RawCaptureResult to CaptureResult metadata, browser URL, focus state, window-to-screen OCR coordinate transformation, Tesseract output parsing, and contour-based text-region detection for the meeting OCR gate. |
| screen-capture-windowing | screenpipe-screen | windows, macos, linux | vision-capture, timeline, performance, privacy | capture-ocr-pipeline, timeline-streaming, privacy-and-redaction, performance-liveness | high | strong | mixed | 14 | 193 | 0 | Window filtering, empty-window regressions, retry policy, URL timing, monitor cache, OCR cache, snapshots, and image comparison. |
@@ -482,7 +482,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su
| engine-api-routes | screenpipe-engine | src/routes/timezone.rs | source | 8 | 0 | 8 |
| engine-api-routes | screenpipe-engine | src/routes/websocket.rs | source | 4 | 0 | 4 |
| engine-workflows | screenpipe-engine | src/routes/workflow_catalog.rs | source | 7 | 0 | 7 |
-| engine-workflows | screenpipe-engine | src/routes/workflow_edits.rs | source | 12 | 0 | 12 |
+| engine-workflows | screenpipe-engine | src/routes/workflow_edits.rs | source | 13 | 0 | 13 |
| engine-workflows | screenpipe-engine | src/routes/workflow_pipeline.rs | source | 4 | 0 | 4 |
| engine-workflows | screenpipe-engine | src/routes/workflows.rs | source | 3 | 0 | 3 |
| engine-config-lifecycle | screenpipe-engine | src/schedule_monitor.rs | source | 8 | 0 | 8 |
diff --git a/docs/pr-assets/sop-share-images/workflow-selected.png b/docs/pr-assets/sop-share-images/workflow-selected.png
new file mode 100644
index 000000000..ee2a53724
Binary files /dev/null and b/docs/pr-assets/sop-share-images/workflow-selected.png differ
diff --git a/packages/workflows-ui/src/fixture-platform.ts b/packages/workflows-ui/src/fixture-platform.ts
index 4db45f0d9..3f61333d0 100644
--- a/packages/workflows-ui/src/fixture-platform.ts
+++ b/packages/workflows-ui/src/fixture-platform.ts
@@ -541,7 +541,8 @@ export function createFixtureWorkflowsPlatform(analysis: WorkflowAnalysis = fixt
const prior = s.sourceIndex === null ? null : workflow.stages[s.sourceIndex];
return { ...(prior ?? { activeMinutes: 0, waitingMinutes: 0, apps: [], confidence: 0, observedOccurrences: 0, observedDays: 0, evidence: [] }),
timing: isUnchangedStage(s, prior ?? undefined) ? prior?.timing : null,
- name: s.name.trim(), description: s.description.trim(), userEdited: true,
+ name: s.name.trim(), description: s.description.trim(), userEdited: prior?.userEdited || !isUnchangedStage(s, prior ?? undefined),
+ imageLayout: s.imageLayout,
procedure: s.procedure.map(p => ({ ...(p.sourceIndex === null ? { quote: "", timestamp: "", app: "" } : prior?.procedure?.[p.sourceIndex] ?? { quote: "", timestamp: "", app: "" }), kind: p.kind, text: p.text.trim(), userEdited: true })),
};
});
diff --git a/packages/workflows-ui/src/model.ts b/packages/workflows-ui/src/model.ts
index b896c315a..fe4b97343 100644
--- a/packages/workflows-ui/src/model.ts
+++ b/packages/workflows-ui/src/model.ts
@@ -92,7 +92,10 @@ export type WorkflowScreenshot = {
dataUrl: string;
};
+export type WorkflowImageLayout = { order: number[]; hidden: number[]; beforeText: boolean };
+
export type WorkflowStage = {
+ imageLayout?: WorkflowImageLayout;
timing?: WorkflowTiming | null;
userEdited?: boolean;
name: string;
diff --git a/packages/workflows-ui/src/workflow-editor.tsx b/packages/workflows-ui/src/workflow-editor.tsx
index dc6fff0fd..9942c2ef3 100644
--- a/packages/workflows-ui/src/workflow-editor.tsx
+++ b/packages/workflows-ui/src/workflow-editor.tsx
@@ -61,7 +61,7 @@ export function WorkflowEditor({ workflow, save, actions, renderSource, stepsOnl
workflow: WorkflowMap;
save: (draft: WorkflowEdit) => Promise<WorkflowMap>;
actions?: ReactNode;
- renderSource?: (stage: WorkflowStage) => ReactNode;
+ renderSource?: (stage: WorkflowStage, onImageLayoutChange?: (layout: NonNullable<WorkflowStage["imageLayout"]>) => void) => ReactNode;
}) {
const key = `screenpipe:workflow-edit:${workflow.id ?? workflow.title}`;
const [initial, setInitial] = useState(() => keyedDraft(workflowEdit(workflow)));
@@ -415,6 +415,7 @@ export function WorkflowEditor({ workflow, save, actions, renderSource, stepsOnl
? workflow.stages[stage.sourceIndex]?.timing : null} />
</div>
</div>
+ <div style={{ display: "flex", flexDirection: "column" }}>
<Text
rich
label={`Step ${index + 1} description`}
@@ -554,7 +555,8 @@ export function WorkflowEditor({ workflow, save, actions, renderSource, stepsOnl
disabled={stage.procedure.length >= 100} onClick={() => addDetail(index)}>
<Plus size={14} aria-hidden="true" />{singleCheck ? "Add expected result" : "Add detail"}
</button>}
- {stage.sourceIndex !== null && workflow.stages[stage.sourceIndex] && <div className={styles.source}>{renderSource?.(workflow.stages[stage.sourceIndex])}</div>}
+ {stage.sourceIndex !== null && workflow.stages[stage.sourceIndex] && <div className={styles.source} style={stage.imageLayout?.beforeText ? { order: -1 } : undefined}>{renderSource?.({ ...workflow.stages[stage.sourceIndex], imageLayout: stage.imageLayout }, layout => stageChange(index, { ...stage, imageLayout: layout }))}</div>}
+ </div>
</article>
</Fragment>
))}
@@ -591,7 +593,7 @@ export function WorkflowDocument({ workflow, actions, sourceLabel, renderSource
workflow: WorkflowMap;
actions?: ReactNode;
sourceLabel?: string;
- renderSource?: (stage: WorkflowStage) => ReactNode;
+ renderSource?: (stage: WorkflowStage, onImageLayoutChange?: (layout: NonNullable<WorkflowStage["imageLayout"]>) => void) => ReactNode;
}) {
return <section className={styles.editor} aria-label="Workflow document">
<header className={styles.toolbar}>
@@ -615,6 +617,7 @@ export function WorkflowDocument({ workflow, actions, sourceLabel, renderSource
<h2 className={`${styles.title} ${styles.readOnlyText}`}>{stage.name}</h2>
<div className={styles.stepActions}><TimingDisclosure hideWhenUnknown showDuration label={`Timing for step ${index + 1}`} value={stage.timing} /></div>
</div>
+ <div style={{ display: "flex", flexDirection: "column" }}>
{stage.description && <WorkflowRichText label={`Step ${index + 1} description`} value={stage.description} />}
<div className={styles.blocks}>
{stage.procedure?.map((detail, detailIndex) => {
@@ -635,7 +638,8 @@ export function WorkflowDocument({ workflow, actions, sourceLabel, renderSource
})}
</div>
{!!stage.apps.length && <p className={styles.stepApps}>{stage.apps.join(" · ")}</p>}
- {renderSource && <div className={styles.source}>{renderSource(stage)}</div>}
+ {renderSource && <div className={styles.source} style={stage.imageLayout?.beforeText ? { order: -1 } : undefined}>{renderSource(stage)}</div>}
+ </div>
</article>)}
</div>
<div className={styles.endpoint}>Ends with<span className={styles.readOnlyText}>{workflow.outcome}</span></div>
diff --git a/packages/workflows-ui/src/workflow-edits.ts b/packages/workflows-ui/src/workflow-edits.ts
index cf654616c..b119adb47 100644
--- a/packages/workflows-ui/src/workflow-edits.ts
+++ b/packages/workflows-ui/src/workflow-edits.ts
@@ -9,6 +9,7 @@ export type ProcedureEdit = {
};
export type StageEdit = {
sourceIndex: number | null;
+ imageLayout?: WorkflowStage["imageLayout"];
name: string;
description: string;
procedure: ProcedureEdit[];
@@ -42,6 +43,7 @@ export function workflowEdit(workflow: WorkflowMap): WorkflowEdit {
sourceIndex,
name: stage.name,
description: stage.description,
+ ...(stage.imageLayout ? { imageLayout: stage.imageLayout } : {}),
procedure: (stage.procedure ?? []).map(({ kind, text }, sourceIndex) => ({
sourceIndex,
kind,
@@ -86,6 +88,7 @@ export function validWorkflowEdit(
(s) =>
text(s.name, true) &&
text(s.description) &&
+ (!s.imageLayout || (typeof s.imageLayout.beforeText === "boolean" && [s.imageLayout.order, s.imageLayout.hidden].every(ids => Array.isArray(ids) && ids.length <= 100 && new Set(ids).size === ids.length && ids.every(id => Number.isSafeInteger(id) && id >= 0)))) &&
Array.isArray(s.procedure) &&
s.procedure.length <= 100 &&
s.procedure.every(
diff --git a/packages/workflows-ui/src/workflow-step-evidence.module.css b/packages/workflows-ui/src/workflow-step-evidence.module.css
index 9069dc88d..472eedfa2 100644
--- a/packages/workflows-ui/src/workflow-step-evidence.module.css
+++ b/packages/workflows-ui/src/workflow-step-evidence.module.css
@@ -3,13 +3,9 @@
.evidence { margin: 12px 0 4px; min-width: 0; }
.previewStatus { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 16px; padding: 16px; margin-bottom: 8px; border: 1px solid var(--line); border-radius: 6px; color: var(--muted); font-size: 12px; }
.previewStatus button { text-decoration: underline; }
-.figure { max-width: 360px; margin: 0 0 20px; }
-.figure.expanded { max-width: 100%; }
-.figure .imageButton { display: block; padding: 0; position: relative; width: 100%; border: 1px solid var(--line); border-radius: 6px; overflow: hidden; background: var(--soft); cursor: zoom-in; }
-.expanded .imageButton { cursor: zoom-out; }
+.figure { position: relative; max-width: 100%; margin: 0 0 20px; }
+.figure .imageButton { display: block; padding: 0; position: relative; width: 100%; border: 1px solid var(--line); border-radius: 6px; overflow: hidden; background: var(--soft); cursor: pointer; }
.imageButton img { display: block; width: 100%; height: auto; object-fit: contain; }
-.zoom { position: absolute; right: 8px; bottom: 8px; display: grid; place-items: center; padding: 6px; background: var(--panel); border: 1px solid var(--line); border-radius: 4px; opacity: 0; }
-.imageButton:hover .zoom, .imageButton:focus-visible .zoom { opacity: 1; }
.figure figcaption { display: flex; justify-content: space-between; align-items: center; gap: 8px; font-size: 11px; line-height: 1.5; color: var(--muted); }
.figure figcaption > span { min-width: 0; overflow-wrap: anywhere; }
.links { display: flex; flex-wrap: wrap; align-items: flex-start; gap: 4px 12px; font-size: 11px; color: var(--muted); }
@@ -21,7 +17,6 @@
/* Reveal playback without competing with the step. */
@media (prefers-reduced-motion: no-preference) {
- .zoom { transition: opacity 150ms ease; }
.player { animation: evidenceReveal 150ms ease-out; }
}
@keyframes evidenceReveal { from { opacity: 0; transform: translateY(-3px); } to { opacity: 1; transform: translateY(0); } }
@@ -32,3 +27,9 @@
.evidence .recording:focus-visible { outline: 1px solid var(--ink); outline-offset: 2px; }
.evidence .recording:disabled { opacity: .5; }
@media (pointer: coarse) { .evidence .recording { width: 44px; height: 44px; } }
+
+.selected .imageButton { outline: 2px solid var(--ink); outline-offset: 2px; }
+.imageActions { position: absolute; top: 8px; right: 8px; display: flex; flex-wrap: wrap; gap: 4px; padding: 4px; max-width: calc(100% - 16px); background: var(--panel); border: 1px solid var(--line); border-radius: 6px; box-shadow: 0 2px 8px #0002; }
+.imageActions button { display: inline-flex; align-items: center; gap: 4px; min-height: 32px; padding: 4px 8px; font: inherit; font-size: 12px; }
+.imageActions button:disabled { opacity: .4; }
+.restore { font: inherit; font-size: 12px; text-decoration: underline; }
diff --git a/packages/workflows-ui/src/workflow-step-evidence.tsx b/packages/workflows-ui/src/workflow-step-evidence.tsx
index 9bb237712..f4a0a19e4 100644
--- a/packages/workflows-ui/src/workflow-step-evidence.tsx
+++ b/packages/workflows-ui/src/workflow-step-evidence.tsx
@@ -3,7 +3,7 @@
"use client";
import { useEffect, useMemo, useRef, useState } from "react";
-import { Loader2, Maximize2, Minimize2, Play, X } from "lucide-react";
+import { Loader2, ArrowUp, ArrowDown, Trash2, Play, X } from "lucide-react";
import type { WorkflowMap, WorkflowStage, WorkflowScreenshot } from "./model";
import type { WorkflowsPlatform } from "./platform";
import { WorkflowReplay } from "./workflow-replay";
@@ -14,10 +14,11 @@ import styles from "./workflow-step-evidence.module.css";
const when = (value: string) => Number.isFinite(Date.parse(value))
? new Date(value).toLocaleString(undefined, { month: "short", day: "numeric", hour: "numeric", minute: "2-digit" }) : "Date unavailable";
-export function WorkflowStepEvidence({ workflow, stage, platform }: {
+export function WorkflowStepEvidence({ workflow, stage, platform, onImageLayoutChange }: {
workflow: WorkflowMap; stage: WorkflowStage; platform: WorkflowsPlatform;
+ onImageLayoutChange?: (layout: NonNullable<WorkflowStage["imageLayout"]>) => void;
}) {
- const [reduced, setReduced] = useState<Set<number>>(() => new Set());
+ const [selectedImage, setSelectedImage] = useState<number | null>(null);
const [previewHeight, setPreviewHeight] = useState(0);
const [selected, setSelected] = useState<WorkflowScreenshot | null>(null);
const [playing, setPlaying] = useState(false);
@@ -31,7 +32,27 @@ export function WorkflowStepEvidence({ workflow, stage, platform }: {
} : stage] }), [workflow, stage, selected]);
const attached = verifiedStageScreenshots(stage);
const preview = useSourceScreenshot(stage, attached.length > 0, platform.loadWorkflowScreenshot);
- const screenshots = attached.length ? attached : preview.image ? [preview.image] : [];
+ const available = attached.length ? attached : preview.image ? [preview.image] : [];
+ const layout = stage.imageLayout ?? { order: [], hidden: [], beforeText: false };
+ const ordered = [...available].sort((a, b) => {
+ const rank = (id: number) => { const index = layout.order.indexOf(id); return index < 0 ? layout.order.length : index; };
+ return rank(a.frameId) - rank(b.frameId);
+ });
+ const screenshots = ordered.filter(image => !layout.hidden.includes(image.frameId));
+ const removeImage = (id: number) => {
+ onImageLayoutChange?.({ ...layout, hidden: [...new Set([...layout.hidden, id])] });
+ setSelectedImage(null);
+ };
+ const moveImage = (index: number, direction: -1 | 1) => {
+ const next = index + direction;
+ if (next < 0 || next >= screenshots.length) {
+ onImageLayoutChange?.({ ...layout, beforeText: direction === -1 });
+ } else {
+ const order = screenshots.map(image => image.frameId);
+ [order[index], order[next]] = [order[next], order[index]];
+ onImageLayoutChange?.({ ...layout, order });
+ }
+ };
const screenshot = screenshots[0];
const canReplay = platform.loadWorkflowRecording && stage.evidence.some(e => !["audio", "meeting"].includes(e.source ?? "") && Number.isFinite(Date.parse(e.timestamp)));
const openRecording = async (capture = screenshot) => {
@@ -54,19 +75,23 @@ export function WorkflowStepEvidence({ workflow, stage, platform }: {
else if (media?.url && platform.releaseWorkflowRecording) void platform.releaseWorkflowRecording(media.url).catch(() => {});
}
};
- return <section ref={preview.ref} onLoad={() => setPreviewHeight(preview.ref.current?.getBoundingClientRect().height ?? 0)} style={!attached.length && !screenshots.length && previewHeight ? { minHeight: previewHeight } : undefined} className={`${styles.evidence} ph-no-capture ph-mask`} aria-label={`References for ${stage.name}`}>
- {!playing && !screenshots.length && <div className={styles.previewStatus} role="status">
+ return <section ref={preview.ref} onKeyDown={event => {
+ if (event.key === "Escape") { setSelectedImage(null); event.stopPropagation(); }
+ }} onBlur={event => { if (event.relatedTarget && !event.currentTarget.contains(event.relatedTarget as Node)) setSelectedImage(null); }} onLoad={() => setPreviewHeight(preview.ref.current?.getBoundingClientRect().height ?? 0)} style={!attached.length && !screenshots.length && previewHeight ? { minHeight: previewHeight } : undefined} className={`${styles.evidence} ph-no-capture ph-mask`} aria-label={`References for ${stage.name}`}>
+ {!playing && !available.length && <div className={styles.previewStatus} role="status">
{preview.status === "loading" ? "Loading source screenshot…" : preview.status === "error" ? "Could not load the source screenshot." : "No screenshot available for this step."}
{preview.canRetry && preview.status !== "loading" && <button type="button" onClick={preview.retry}>Retry screenshot</button>}
</div>}
{!playing && screenshots.map((capture, index) => {
- const expanded = !reduced.has(capture.frameId);
const suffix = screenshots.length > 1 ? ` ${index + 1}` : "";
- return <EvidenceFigure key={`${capture.frameId}:${capture.timestamp}`} capture={capture} stage={stage} platform={platform} suffix={suffix} expanded={expanded} opening={opening}
- toggle={() => setReduced(previous => { const next = new Set(previous); if (expanded) next.add(capture.frameId); else next.delete(capture.frameId); return next; })}
+ return <EvidenceFigure key={`${capture.frameId}:${capture.timestamp}`} capture={capture} stage={stage} platform={platform} suffix={suffix} selected={selectedImage === capture.frameId} opening={opening}
+ select={() => setSelectedImage(capture.frameId)}
+ actions={onImageLayoutChange ? { remove: () => removeImage(capture.frameId), up: () => moveImage(index, -1), down: () => moveImage(index, 1),
+ upDisabled: index === 0 && layout.beforeText, downDisabled: index === screenshots.length - 1 && !layout.beforeText } : undefined}
replay={canReplay || !!platform.openCapturedMoment ? () => void openRecording(capture) : undefined}
resolved={capture === preview.image} />;
})}
+ {!playing && onImageLayoutChange && layout.hidden.length > 0 && <button type="button" className={styles.restore} onClick={() => onImageLayoutChange({ ...layout, hidden: [] })}>Restore deleted images</button>}
{playing && <div ref={playerRef} className={styles.player}>
<button type="button" className={styles.close} aria-label={`Close recording for ${stage.name}`} title="Close recording" onClick={() => setPlaying(false)}><X size={15} /></button>
<WorkflowReplay workflow={replay} loadRecording={platform.loadWorkflowRecording} releaseRecording={platform.releaseWorkflowRecording} openCapturedMoment={platform.openCapturedMoment} />
@@ -78,9 +103,10 @@ export function WorkflowStepEvidence({ workflow, stage, platform }: {
</section>;
}
-function EvidenceFigure({ capture, stage, platform, suffix, expanded, opening, toggle, replay, resolved }: {
+function EvidenceFigure({ capture, stage, platform, suffix, selected, opening, select, actions, replay, resolved }: {
capture: WorkflowScreenshot; stage: WorkflowStage; platform: WorkflowsPlatform; suffix: string;
- expanded: boolean; opening: boolean; toggle: () => void; replay?: () => void; resolved: boolean;
+ selected: boolean; opening: boolean; select: () => void; replay?: () => void; resolved: boolean;
+ actions?: { remove: () => void; up: () => void; down: () => void; upDisabled: boolean; downDisabled: boolean };
}) {
const load = resolved ? undefined : platform.loadWorkflowScreenshot;
const preview = useSourceScreenshot(stage, !load, load, capture);
@@ -89,14 +115,19 @@ function EvidenceFigure({ capture, stage, platform, suffix, expanded, opening, t
const [failed, setFailed] = useState(false);
const [ratio, setRatio] = useState("16 / 10");
useEffect(() => setFailed(false), [url]);
- return <figure ref={preview.ref} className={`${styles.figure} ${expanded ? styles.expanded : ""}`}>
- {url && !failed ? <button className={styles.imageButton} type="button" aria-label={`${expanded ? "Reduce" : "Enlarge"} screenshot${suffix} for ${stage.name}`} aria-expanded={expanded} onClick={toggle}>
+ return <figure ref={preview.ref} className={`${styles.figure} ${selected ? styles.selected : ""}`}>
+ {url && !failed ? <button className={styles.imageButton} type="button" aria-label={`Select screenshot${suffix} for ${stage.name}`} aria-pressed={selected} onClick={select}
+ onKeyDown={event => { if (selected && actions && ["Delete", "Backspace"].includes(event.key)) { event.preventDefault(); event.stopPropagation(); actions.remove(); } }}>
<img src={url} alt={`Captured reference${suffix} for ${stage.name}`} loading="lazy" decoding="async" draggable={false} data-lm-disable="true" onLoad={event => { const image = event.currentTarget; if (image.naturalWidth && image.naturalHeight) setRatio(`${image.naturalWidth} / ${image.naturalHeight}`); }} onError={() => setFailed(true)} />
- <span className={styles.zoom}>{expanded ? <Minimize2 size={15} /> : <Maximize2 size={15} />}</span>
</button> : <div className={styles.previewStatus} role="status" style={preview.status === "loading" && !failed ? { aspectRatio: ratio } : undefined}>
{failed || preview.status === "error" ? "Could not load the source screenshot." : preview.status === "loading" ? "Loading source screenshot…" : "This screenshot is no longer available."}
{(failed || preview.status === "error") && load && <button onClick={() => { setFailed(false); preview.retry(); }}>Retry screenshot</button>}
</div>}
+ {selected && actions && <div className={styles.imageActions} role="toolbar" aria-label="Selected image">
+ <button type="button" onClick={actions.up} disabled={actions.upDisabled} aria-label="Move image up"><ArrowUp size={15} />Move up</button>
+ <button type="button" onClick={actions.down} disabled={actions.downDisabled} aria-label="Move image down"><ArrowDown size={15} />Move down</button>
+ <button type="button" onClick={actions.remove} aria-label="Delete image"><Trash2 size={15} />Delete</button>
+ </div>}
<figcaption><span>{!capture.visualVerified ? "Source screenshot · " : ""}{capture.app} · {when(capture.timestamp)}</span>
{replay && <button className={styles.recording} title="Open recording" type="button" disabled={opening} onClick={replay} aria-label={`${platform.openCapturedMoment ? "Open" : "View"} recording${suffix} for ${stage.name}`}>{opening ? <Loader2 size={15} aria-hidden="true" /> : <Play size={15} aria-hidden="true" />}</button>}
</figcaption>
diff --git a/packages/workflows-ui/src/workflows-app.tsx b/packages/workflows-ui/src/workflows-app.tsx
index dab644a88..22f3f8e97 100644
--- a/packages/workflows-ui/src/workflows-app.tsx
+++ b/packages/workflows-ui/src/workflows-app.tsx
@@ -728,7 +728,7 @@ export function WorkflowDetail({ editorOptions, sourceLabel, observationsAvailab
return (
<>
<button className={styles.backButton} onClick={() => navigate("workflows")}><ArrowLeft size={14} />All workflows</button>
- {saveEdits ? <WorkflowEditor {...editorOptions} key={workflow.id || workflow.title} workflow={workflow} save={saveEdits} actions={workflowActions} renderSource={observationsAvailable ? stage => <WorkflowStepEvidence workflow={workflow} stage={stage} platform={platform} /> : undefined} /> : <WorkflowDocument workflow={workflow} actions={workflowActions} sourceLabel={sourceLabel}
+ {saveEdits ? <WorkflowEditor {...editorOptions} key={workflow.id || workflow.title} workflow={workflow} save={saveEdits} actions={workflowActions} renderSource={observationsAvailable ? (stage, onImageLayoutChange) => <WorkflowStepEvidence workflow={workflow} stage={stage} platform={platform} onImageLayoutChange={onImageLayoutChange} /> : undefined} /> : <WorkflowDocument workflow={workflow} actions={workflowActions} sourceLabel={sourceLabel}
renderSource={observationsAvailable ? stage => <WorkflowStepEvidence workflow={workflow} stage={stage} platform={platform} /> : undefined} />}
<div className={styles.workflowNotes}>
<p className={styles.workflowReviewState}>{workflow.userEditedAt ? ui("Edited by you · sources kept as references") : workflow.evidenceStatus === "supported-steps" ? ui("Source-backed steps · not execution-tested") : ui("Needs review")}</p>