Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Simplify image edits and link per-edit references

Bitti9c0bdd5c8904
diff --git a/src-tauri/src/project/commands/images.rs b/src-tauri/src/project/commands/images.rs
index 23e23a2..7e7c254 100644
--- a/src-tauri/src/project/commands/images.rs
+++ b/src-tauri/src/project/commands/images.rs
@@ -32,6 +32,7 @@ fn new_scene(prompt: &str) -> ImageScene {
name: "Image".into(),
description: prompt.into(),
text: String::new(),
+ reference_ids: None,
r#box: None,
colors: vec![],
}],
@@ -232,6 +233,7 @@ pub(super) fn apply(
r#box,
colors,
before,
+ reference_ids,
} => {
if !matches!(kind.as_str(), "object" | "text" | "group" | "background") {
return Err("New image nodes must be objects, text, groups or backgrounds.".into());
@@ -243,6 +245,7 @@ pub(super) fn apply(
name: name.clone(),
description: description.clone().unwrap_or_default(),
text: text.clone().unwrap_or_default(),
+ reference_ids: reference_ids.clone(),
r#box: r#box.clone(),
colors: colors.clone().unwrap_or_default(),
};
@@ -255,6 +258,7 @@ pub(super) fn apply(
text,
r#box,
colors,
+ reference_ids,
} => {
let at = index(&scene, id)?;
ensure_any(
@@ -264,6 +268,7 @@ pub(super) fn apply(
text.is_some(),
r#box.is_changed(),
colors.is_some(),
+ reference_ids.is_some(),
],
"Image node update has no fields to change.",
)?;
@@ -284,6 +289,9 @@ pub(super) fn apply(
if let Some(value) = colors {
scene.nodes[at].colors = value.clone();
}
+ if let Some(value) = reference_ids {
+ scene.nodes[at].reference_ids = Some(value.clone());
+ }
}
ProjectCommand::ImageNodeMove { id, parent, before } => {
let at = non_root(&scene, id)?;
diff --git a/src-tauri/src/project/commands/types.rs b/src-tauri/src/project/commands/types.rs
index bf0181a..be15e7b 100644
--- a/src-tauri/src/project/commands/types.rs
+++ b/src-tauri/src/project/commands/types.rs
@@ -47,6 +47,8 @@ pub enum ProjectCommand {
colors: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
before: Option<String>,
+ #[serde(rename = "referenceIds", default, skip_serializing_if = "Option::is_none")]
+ reference_ids: Option<Vec<String>>,
},
#[serde(rename = "image.node.set")]
ImageNodeSet {
@@ -61,6 +63,8 @@ pub enum ProjectCommand {
r#box: Patch<crate::project::image::ImageBox>,
#[serde(default, skip_serializing_if = "Option::is_none")]
colors: Option<Vec<String>>,
+ #[serde(rename = "referenceIds", default, skip_serializing_if = "Option::is_none")]
+ reference_ids: Option<Vec<String>>,
},
#[serde(rename = "image.node.move")]
ImageNodeMove {
diff --git a/src-tauri/src/project/image.rs b/src-tauri/src/project/image.rs
index faef53e..b536256 100644
--- a/src-tauri/src/project/image.rs
+++ b/src-tauri/src/project/image.rs
@@ -46,6 +46,8 @@ pub struct ImageNode {
pub name: String,
pub description: String,
pub text: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub reference_ids: Option<Vec<String>>,
pub r#box: Option<ImageBox>,
pub colors: Vec<String>,
}
@@ -111,6 +113,7 @@ impl ImageScene {
"root" | "group" | "object" | "text" | "background"
)
|| node.colors.len() > 16
+ || node.reference_ids.as_ref().is_some_and(|ids| ids.len() > 100 || ids.iter().any(String::is_empty))
|| node.colors.iter().any(|color| {
color.len() != 7
|| !color.starts_with('#')
diff --git a/src-tauri/src/project/validation/assets.rs b/src-tauri/src/project/validation/assets.rs
index d77eacc..91619f5 100644
--- a/src-tauri/src/project/validation/assets.rs
+++ b/src-tauri/src/project/validation/assets.rs
@@ -19,7 +19,7 @@ pub(super) fn validate(config: &mut ProjectConfig) -> Result<(), String> {
}
super::references::validate_references(&mut snapshot.references)?;
let ids = unique_ids(snapshot.references.iter().map(|reference| reference.id.as_str()), "image snapshot reference")?;
- if snapshot.scene.reference_ids.iter().any(|id| !ids.contains(id.as_str())) {
+ if snapshot.scene.reference_ids.iter().chain(snapshot.scene.nodes.iter().flat_map(|node| node.reference_ids.iter().flatten())).any(|id| !ids.contains(id.as_str())) {
return Err("Image generation snapshot has a missing reference.".into());
}
}
diff --git a/src-tauri/src/slopfab/generation.rs b/src-tauri/src/slopfab/generation.rs
index 97391be..a538599 100644
--- a/src-tauri/src/slopfab/generation.rs
+++ b/src-tauri/src/slopfab/generation.rs
@@ -47,6 +47,8 @@ fn run_edit_sequence(
if item.cancel.load(Ordering::Acquire) { return Err("Image editing cancelled.".into()); }
let mut request = item.request.clone();
request.prompt = step.prompt.clone();
+ if let Some(paths) = &step.reference_paths { request.reference_paths = paths.clone(); }
+ if let Some(refmods) = &step.refmods { request.refmods = refmods.clone(); }
request.image_edit.as_mut().unwrap().edits = vec![step.clone()];
request.image_edit_pixels = pixels.take();
let sink = item.events.clone();
diff --git a/src-tauri/src/slopfab/generation_tests.rs b/src-tauri/src/slopfab/generation_tests.rs
index 867b927..952b800 100644
--- a/src-tauri/src/slopfab/generation_tests.rs
+++ b/src-tauri/src/slopfab/generation_tests.rs
@@ -54,6 +54,34 @@ fn sequential_image_edits_feed_previous_pixels_and_report_combined_progress() {
assert_eq!(progress[1].elapsed_seconds, 2.0);
}
+#[test]
+fn image_edits_use_their_own_reference_pictures_and_refmods() {
+ let mut item = item();
+ item.request.reference_paths = vec!["legacy.png".into()];
+ item.request.refmods = vec![serde_json::from_value(serde_json::json!({"path":"legacy.safetensors","strength":1.0,"copies":1})).unwrap()];
+ let steps = &mut item.request.image_edit.as_mut().unwrap().edits;
+ steps[0].reference_paths = Some(vec!["source.png".into(), "balloon.png".into()]);
+ steps[0].refmods = Some(vec![serde_json::from_value(serde_json::json!({"path":"balloon.safetensors","strength":0.75,"copies":2})).unwrap()]);
+ steps[1].reference_paths = Some(vec!["source.png".into()]);
+ steps[1].refmods = Some(vec![]);
+ let mut calls = 0;
+ run_edit_sequence(&item, |stage| {
+ calls += 1;
+ if calls == 1 {
+ assert_eq!(stage.request.reference_paths, ["source.png", "balloon.png"]);
+ assert_eq!(stage.request.refmods[0].path, "balloon.safetensors");
+ assert_eq!(stage.request.refmods[0].strength, 0.75);
+ assert_eq!(stage.request.refmods[0].copies, 2);
+ } else {
+ assert_eq!(stage.request.reference_paths, ["source.png"]);
+ assert!(stage.request.refmods.is_empty());
+ assert!(stage.request.image_edit_pixels.is_some());
+ }
+ Ok(result(calls))
+ }).unwrap();
+ assert_eq!(calls, 2);
+}
+
#[test]
fn cancellation_and_failures_do_not_publish_partial_image_edits() {
for cancel in [false, true] {
diff --git a/src-tauri/src/slopfab/planning.rs b/src-tauri/src/slopfab/planning.rs
index d4a015f..9ed16a5 100644
--- a/src-tauri/src/slopfab/planning.rs
+++ b/src-tauri/src/slopfab/planning.rs
@@ -59,7 +59,7 @@ pub fn resolve_plan(
pub(super) fn validate_generation_controls(request: &GenerationRequest) -> Result<(), String> {
if let Some(edit) = &request.image_edit {
if !request.still_image || request.frames != 1 || request.continuation_relative_path.is_some() || request.continuation_path.is_some() || request.video_transition.is_some()
- || edit.edits.is_empty() || edit.edits.len() > 499 || request.canvas_width > 8192 || request.canvas_height > 8192
+ || edit.edits.is_empty() || edit.edits.len() > 500 || request.canvas_width > 8192 || request.canvas_height > 8192
|| edit.edits.iter().any(|step| step.prompt.trim().is_empty() || step.x < 0 || step.y < 0 || step.width <= 0 || step.height <= 0
|| i64::from(step.x) + i64::from(step.width) > i64::from(request.canvas_width)
|| i64::from(step.y) + i64::from(step.height) > i64::from(request.canvas_height)) {
diff --git a/src-tauri/src/slopfab/tests.rs b/src-tauri/src/slopfab/tests.rs
index 36b7ee0..32bf424 100644
--- a/src-tauri/src/slopfab/tests.rs
+++ b/src-tauri/src/slopfab/tests.rs
@@ -16,7 +16,7 @@ fn image_edit_recipe_disables_motion_cache_and_keeps_source_geometry() {
canvas_width: 65, canvas_height: 41,
image_edit_pixels: Some(std::sync::Arc::new(vec![100; 65 * 41 * 3])),
image_edit: Some(types::ImageEditRequest { source_relative_path: "media/source.png".into(), edits: vec![
- types::ImageEditStep { prompt: "A red vase".into(), x: 30, y: 10, width: 35, height: 31 }
+ types::ImageEditStep { prompt: "A red vase".into(), x: 30, y: 10, width: 35, height: 31, reference_paths: None, refmods: None }
] }), ..Default::default()
};
validate_generation_controls(&request).unwrap();
diff --git a/src-tauri/src/slopfab/types.rs b/src-tauri/src/slopfab/types.rs
index 2fc526e..be3e2c3 100644
--- a/src-tauri/src/slopfab/types.rs
+++ b/src-tauri/src/slopfab/types.rs
@@ -78,12 +78,17 @@ pub struct ImageEditRequest {
pub edits: Vec<ImageEditStep>,
}
#[derive(Debug, Clone, Deserialize)]
+#[serde(rename_all = "camelCase")]
pub struct ImageEditStep {
pub prompt: String,
pub x: i32,
pub y: i32,
pub width: i32,
pub height: i32,
+ #[serde(default)]
+ pub reference_paths: Option<Vec<String>>,
+ #[serde(default)]
+ pub refmods: Option<Vec<RefmodInput>>,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
diff --git a/src-tauri/src/tests/images.rs b/src-tauri/src/tests/images.rs
index b56534b..e842cc9 100644
--- a/src-tauri/src/tests/images.rs
+++ b/src-tauri/src/tests/images.rs
@@ -167,7 +167,9 @@ fn image_tree_survives_folder_creation_save_and_reopen() {
#[test]
fn image_generation_history_survives_native_save_and_reopen_and_validates_snapshots() {
- let snapshot: serde_json::Value = serde_json::from_str(include_str!("../../../fixtures/image-generation-snapshot.json")).unwrap();
+ let mut snapshot: serde_json::Value = serde_json::from_str(include_str!("../../../fixtures/image-generation-snapshot.json")).unwrap();
+ snapshot["scene"]["nodes"][1]["referenceIds"] = snapshot["scene"]["referenceIds"].clone();
+ snapshot["scene"]["referenceIds"] = serde_json::json!([]);
let expected: crate::project::image::ImageGenerationSnapshot = serde_json::from_value(snapshot.clone()).unwrap();
let mut config = image_fixture();
config.assets.push(serde_json::from_value(serde_json::json!({
diff --git a/src/components/workspace/ImageEditor.test.tsx b/src/components/workspace/ImageEditor.test.tsx
index ce4b677..662c13e 100644
--- a/src/components/workspace/ImageEditor.test.tsx
+++ b/src/components/workspace/ImageEditor.test.tsx
@@ -1,7 +1,7 @@
// @vitest-environment jsdom
import "@testing-library/jest-dom/vitest";
import { useState } from "react";
-import { cleanup, createEvent, fireEvent, render, screen, within } from "@testing-library/react";
+import { cleanup, createEvent, fireEvent, render, screen, waitFor, within } from "@testing-library/react";
import { afterEach, expect, it, vi } from "vitest";
import { ImageEditor } from "./ImageEditor";
import { choose, comboValue } from "./comboTestUtils";
@@ -9,10 +9,14 @@ import { imageGenerationSnapshotSchema, parseProjectConfig, type ProjectConfig }
import { imageGenerationSnapshot } from "../../lib/imageHistory";
import { defaultGeneratorTemplate, saveGeneratorTemplateSettings } from "../../lib/settings";
import { imageScenePrompt } from "../../lib/imageScene";
+import { compileImageEdits } from "../../lib/imageEditing";
+import { invoke } from "@tauri-apps/api/core";
+import * as persistence from "../../lib/persistence";
import fixture from "../../../fixtures/project-v1-image.json";
import snapshotFixture from "../../../fixtures/image-generation-snapshot.json";
-afterEach(() => { cleanup(); localStorage.clear(); });
+vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() }));
+afterEach(() => { cleanup(); localStorage.clear(); vi.restoreAllMocks(); });
function setup(initial = parseProjectConfig(fixture)) {
let latest: ProjectConfig = initial;
function Harness() {
@@ -27,7 +31,65 @@ function setup(initial = parseProjectConfig(fixture)) {
/* Undo lives in the title bar; inside the editor it is Ctrl+Z. */
const undoImageEdit = () => fireEvent.keyDown(screen.getByRole("tree", { name: "Image nodes" }), { key: "z", code: "KeyZ", ctrlKey: true });
-it("starts an image root from the thumbnail Edit menu and hides root text controls", () => {
+it("adds image files in edit mode and saves whole-image and drawn edits with separate references", async () => {
+ vi.spyOn(persistence, "isTauri").mockReturnValue(true);
+ vi.spyOn(persistence, "readMediaFileUrl").mockResolvedValue(null);
+ const initial = parseProjectConfig(fixture);
+ initial.references = ["Mood", "Balloon"].map((name) => ({ id: name, name, kind: "text", description: name, intendedUse: [], createdAt: initial.createdAt }));
+ const current = setup(initial);
+ const addFile = () => {
+ fireEvent.contextMenu(screen.getByLabelText("Generated images"));
+ fireEvent.click(screen.getByRole("menuitem", { name: "Add image file" }));
+ };
+ vi.mocked(invoke).mockResolvedValueOnce(null);
+ addFile();
+ await waitFor(() => expect(invoke).toHaveBeenCalledWith("open_image_source", { folderPath: "D:/Images" }));
+ expect(current().imageScene).toEqual(initial.imageScene);
+ const source = { name: "Imported", relativePath: "media/imported/source.png", width: 1000, height: 800 };
+ vi.mocked(invoke).mockResolvedValueOnce(source);
+ addFile();
+ await waitFor(() => expect(current().imageScene?.sourceImage).toEqual(source));
+ expect(screen.queryByRole("combobox", { name: "Type" })).not.toBeInTheDocument();
+ fireEvent.change(screen.getByLabelText("Edit prompt"), { target: { value: "Make the lighting warmer" } });
+ fireEvent.click(screen.getByRole("checkbox", { name: "Mood" }));
+ expect(compileImageEdits(current()).edits[0]).toMatchObject({ x: 0, y: 0, width: 1000, height: 800 });
+ const canvas = screen.getByLabelText("Image placement canvas");
+ Object.defineProperty(canvas, "setPointerCapture", { value: vi.fn() });
+ Object.defineProperty(canvas, "getBoundingClientRect", { value: () => ({ left: 0, top: 0, width: 1000, height: 800 }) });
+ const pointer = (type: "pointerDown" | "pointerMove" | "pointerUp", x: number, y: number) => {
+ const event = createEvent[type](canvas);
+ Object.defineProperties(event, { button: { value: 0 }, buttons: { value: type === "pointerUp" ? 0 : 1 }, pointerId: { value: 1 }, clientX: { value: x }, clientY: { value: y } });
+ fireEvent(canvas, event);
+ };
+ for (const key of ["o", "t", "g"]) fireEvent.keyDown(canvas, { key });
+ expect(screen.getByRole("radio", { name: "Select" })).toHaveAttribute("aria-checked", "true");
+ fireEvent.keyDown(canvas, { key: "e" });
+ expect(screen.getByRole("radio", { name: "Draw edit" })).toHaveAttribute("aria-checked", "true");
+ pointer("pointerDown", 100, 80); pointer("pointerMove", 400, 320); pointer("pointerUp", 400, 320);
+ expect(current().imageScene!.nodes[1]).toMatchObject({ name: "Edit", box: { x: 100, y: 100, width: 300, height: 300 }, referenceIds: [] });
+ expect(screen.getByRole("checkbox", { name: "Mood" })).not.toBeChecked();
+ fireEvent.change(screen.getByLabelText("Edit prompt"), { target: { value: "Add a red balloon" } });
+ fireEvent.click(screen.getByRole("checkbox", { name: "Balloon" }));
+ const edits = compileImageEdits(current()).edits;
+ expect(edits.map((edit) => edit.references.map((reference) => reference.id))).toEqual([["Mood"], ["Balloon"]]);
+ const reopened = parseProjectConfig(JSON.parse(JSON.stringify(current())));
+ expect(reopened.imageScene).toEqual(current().imageScene);
+ expect(reopened.assets.at(-1)!.imageGeneration!.references.map((reference) => reference.id)).toEqual(["Mood", "Balloon"]);
+ fireEvent.contextMenu(screen.getByLabelText("Generated images"));
+ fireEvent.click(screen.getByRole("menuitem", { name: "New empty image" }));
+ fireEvent.click(screen.getByRole("button", { name: "View Editing Imported" }));
+ expect(screen.getByLabelText("Edit prompt")).toHaveValue("Make the lighting warmer");
+ fireEvent.click(screen.getByRole("button", { name: "Edit" }));
+ expect(screen.getByLabelText("Edit prompt")).toHaveValue("Add a red balloon");
+ expect(screen.getByRole("checkbox", { name: "Balloon" })).toBeChecked();
+ const beforeFailure = current();
+ vi.mocked(invoke).mockRejectedValueOnce(new Error("Image file is unreadable"));
+ addFile();
+ expect(await screen.findByText(/Image file is unreadable/)).toBeInTheDocument();
+ expect(current()).toEqual(beforeFailure);
+});
+
+it("starts an image root from the thumbnail Edit menu with edit tools and no type selector", () => {
const initial = parseProjectConfig(fixture);
initial.assets = [{ id: "saved", name: "Saved", kind: "image", relativePath: "media/generated/saved.jpg", mimeType: "image/jpeg", width: 101, height: 77, createdAt: initial.createdAt }];
const current = setup(initial);
@@ -36,12 +98,15 @@ it("starts an image root from the thumbnail Edit menu and hides root text contro
expect(current().imageScene).toMatchObject({ rootType: "image", sourceImage: { relativePath: "media/generated/saved.jpg", width: 101, height: 77 }, outputAssetId: expect.stringMatching(/^image-draft-/) });
expect(current().imageScene!.nodes).toHaveLength(1);
expect(screen.queryByLabelText("Prompt (high-level description)")).not.toBeInTheDocument();
- expect(screen.getByRole("button", { name: "Open image" })).toBeInTheDocument();
+ expect(screen.queryByRole("combobox", { name: "Type" })).not.toBeInTheDocument();
+ expect(screen.getByLabelText("Edit prompt")).toHaveValue("");
+ expect(within(screen.getByRole("radiogroup", { name: "Canvas tool" })).getAllByRole("radio").map((tool) => tool.getAttribute("aria-label"))).toEqual(["Select", "Draw edit"]);
const draft = screen.getByRole("button", { name: "View Editing Saved" });
expect(draft).toHaveAttribute("aria-pressed", "true");
fireEvent.contextMenu(screen.getByRole("button", { name: "Image" }));
- fireEvent.click(screen.getByRole("menuitem", { name: "New Object" }));
- fireEvent.change(screen.getByLabelText("Description"), { target: { value: "A red balloon" } });
+ expect(screen.getAllByRole("menuitem").filter((item) => item.textContent?.startsWith("New ")).map((item) => item.textContent)).toEqual(["New Edit"]);
+ fireEvent.click(screen.getByRole("menuitem", { name: "New Edit" }));
+ fireEvent.change(screen.getByLabelText("Edit prompt"), { target: { value: "A red balloon" } });
fireEvent.click(screen.getByRole("button", { name: "View Saved" }));
expect(draft).toHaveAttribute("aria-pressed", "false");
fireEvent.click(draft);
@@ -49,10 +114,12 @@ it("starts an image root from the thumbnail Edit menu and hides root text contro
expect(current().assets.filter((asset) => asset.imageDraft)).toHaveLength(1);
expect(parseProjectConfig(JSON.parse(JSON.stringify(current()))).assets).toEqual(current().assets);
expect(screen.getByLabelText("Image placement canvas").parentElement!.style.getPropertyValue("--image-ratio")).toBe(String(101 / 77));
- choose("Type", "Prompt");
+ fireEvent.contextMenu(screen.getByLabelText("Generated images"));
+ fireEvent.click(screen.getByRole("menuitem", { name: "New empty image" }));
expect(screen.getByLabelText("Prompt (high-level description)")).toBeInTheDocument();
- choose("Type", "Image");
- expect(current().imageScene!.sourceImage).toBeNull();
+ expect(screen.getByRole("radio", { name: "Draw object" })).toBeInTheDocument();
+ fireEvent.click(draft);
+ expect(current().imageScene!.sourceImage?.name).toBe("Saved");
});
it("restores the hierarchy, prompts, settings and generator when selecting a saved result", () => {
@@ -148,7 +215,7 @@ it("creates a blank draft from the image bar and preserves it while browsing", (
const current = setup(initial);
fireEvent.contextMenu(screen.getByRole("button", { name: "View Saved" }));
fireEvent.click(screen.getByRole("menuitem", { name: "New empty image" }));
- expect(comboValue(screen.getByRole("combobox", { name: "Type" }))).toBe("prompt");
+ expect(screen.queryByRole("combobox", { name: "Type" })).not.toBeInTheDocument();
expect(screen.getByLabelText("Prompt (high-level description)")).toHaveValue("");
expect(current().imageScene!.nodes).toHaveLength(1);
expect(screen.getByRole("button", { name: "View New image" })).toHaveAttribute("aria-pressed", "true");
diff --git a/src/components/workspace/ImageEditor.tsx b/src/components/workspace/ImageEditor.tsx
index 78b0597..6e130ea 100644
--- a/src/components/workspace/ImageEditor.tsx
+++ b/src/components/workspace/ImageEditor.tsx
@@ -143,8 +143,11 @@ export function ImageEditor({ config, folderPath, onChange: changeConfig, onGene
const patchNode = (patch: Partial<ImageNode>) => commit({ ...scene, nodes: scene.nodes.map((node) => node.id === selected.id ? { ...node, ...patch } : node) });
const insertParent = selected.kind === "object" || selected.kind === "text" ? selected.parentId ?? root.id : selected.id;
const add = (kind: Exclude<ImageNode["kind"], "root">, box?: ImageBox) => {
- const next = addImageNode(scene, insertParent, kind, box); commit(next); setSelection(next.nodes.at(-1)!.id);
- setCollapsed((current) => { const next = new Set(current); next.delete(insertParent); return next; });
+ const parent = imageRoot ? root.id : insertParent;
+ const next = addImageNode(scene, parent, imageRoot ? "object" : kind, box);
+ if (imageRoot) Object.assign(next.nodes.at(-1)!, { name: "Edit", referenceIds: [] });
+ commit(next); setSelection(next.nodes.at(-1)!.id);
+ setCollapsed((current) => { const next = new Set(current); next.delete(parent); return next; });
};
const attempt = (action: () => void | Promise<unknown>) => {
setError(null);
@@ -335,15 +338,18 @@ export function ImageEditor({ config, folderPath, onChange: changeConfig, onGene
/* Canvas tools: a radio group of icon toggles with one-letter keys. */
const TOOLS = [
{ id: null, label: "Select", key: "V", icon: <Cursor16 />, onIcon: <Cursor16Filled /> },
- { id: "object", label: "Draw object", key: "O", icon: <Cube16 />, onIcon: <Cube16Filled /> },
- { id: "text", label: "Draw text", key: "T", icon: <Text16 />, onIcon: <Text16Filled /> },
- { id: "group", label: "Draw group", key: "G", icon: <Group16 />, onIcon: <Group16Filled /> },
+ ...(imageRoot ? [{ id: "object", label: "Draw edit", key: "E", icon: <Edit16 />, onIcon: <Edit16 /> }] as const : [
+ { id: "object", label: "Draw object", key: "O", icon: <Cube16 />, onIcon: <Cube16Filled /> },
+ { id: "text", label: "Draw text", key: "T", icon: <Text16 />, onIcon: <Text16Filled /> },
+ { id: "group", label: "Draw group", key: "G", icon: <Group16 />, onIcon: <Group16Filled /> },
+ ] as const),
] as const;
const chooseTool = (id: typeof drawKind) => { setDrawKind(id); if (id) setBoxes(true); };
useShortcut("V", () => chooseTool(null), { scope: editorRoot });
- useShortcut("O", () => chooseTool("object"), { scope: editorRoot });
- useShortcut("T", () => chooseTool("text"), { scope: editorRoot });
- useShortcut("G", () => chooseTool("group"), { scope: editorRoot });
+ useShortcut("E", () => chooseTool("object"), { scope: editorRoot, enabled: imageRoot });
+ useShortcut("O", () => chooseTool("object"), { scope: editorRoot, enabled: !imageRoot });
+ useShortcut("T", () => chooseTool("text"), { scope: editorRoot, enabled: !imageRoot });
+ useShortcut("G", () => chooseTool("group"), { scope: editorRoot, enabled: !imageRoot });
useShortcut("Escape", () => { if (!drawKind) return false; chooseTool(null); }, { scope: editorRoot });
/* Zoom: Fit is the frame fitted to the viewport (100% of the fit); the
@@ -420,7 +426,7 @@ export function ImageEditor({ config, folderPath, onChange: changeConfig, onGene
event.stopPropagation();
if (event.key === "Enter" && !event.nativeEvent.isComposing) { event.preventDefault(); finishRename(true); }
if (event.key === "Escape") { event.preventDefault(); finishRename(false); }
- }} /> : <button className="image-tree__select" tabIndex={selected.id === node.id ? 0 : -1} data-tooltip={node.name} onFocus={() => setSelection(node.id)} onClick={() => setSelection(node.id)}><span className="image-tree__icon" aria-hidden="true">{node.kind === "text" ? <Text16 /> : node.kind === "root" ? <Image16 /> : node.kind === "group" ? <Group16 /> : <Cube16 />}</span><span>{node.name}</span></button>}
+ }} /> : <button className="image-tree__select" tabIndex={selected.id === node.id ? 0 : -1} data-tooltip={node.name} onFocus={() => setSelection(node.id)} onClick={() => setSelection(node.id)}><span className="image-tree__icon" aria-hidden="true">{node.kind === "root" ? <Image16 /> : imageRoot ? <Edit16 /> : node.kind === "text" ? <Text16 /> : node.kind === "group" ? <Group16 /> : <Cube16 />}</span><span>{node.name}</span></button>}
</div>
{open && children.length > 0 && <div role="group">{children.map((child) => tree(child, depth + 1))}</div>}
</div>;
@@ -434,10 +440,12 @@ export function ImageEditor({ config, folderPath, onChange: changeConfig, onGene
</aside>
<Splitter {...treePane.splitterProps} aria-label="Resize hierarchy" />
{contextMenu && <HierarchyContextMenu {...contextMenu} onClose={closeContextMenu} items={[
- { label: "New Object", icon: <Cube16 />, action: () => attempt(() => add("object")) },
- { label: "New Text", icon: <Text16 />, action: () => attempt(() => add("text")) },
- { label: "New Group", icon: <FolderAdd16 />, action: () => attempt(() => add("group")) },
- { label: "New Background", icon: <Image16 />, action: () => attempt(() => add("background")) },
+ ...(imageRoot ? [{ label: "New Edit", icon: <Edit16 />, action: () => attempt(() => add("object")) }] : [
+ { label: "New Object", icon: <Cube16 />, action: () => attempt(() => add("object")) },
+ { label: "New Text", icon: <Text16 />, action: () => attempt(() => add("text")) },
+ { label: "New Group", icon: <FolderAdd16 />, action: () => attempt(() => add("group")) },
+ { label: "New Background", icon: <Image16 />, action: () => attempt(() => add("background")) },
+ ]),
{ label: "Rename", icon: <Rename16 />, shortcut: "F2", separator: true, action: startRename },
{ label: "Copy", icon: <Copy16 />, shortcut: "Ctrl+C", separator: true, disabled: selected.kind === "root", action: () => copyImageNode(scene, selected.id) },
{ label: "Paste", icon: <Paste16 />, shortcut: "Ctrl+V", disabled: !clipboard, action: () => attempt(paste) },
@@ -449,6 +457,10 @@ export function ImageEditor({ config, folderPath, onChange: changeConfig, onGene
undo.current = []; redo.current = []; pointer.current = null; setDraftBox(null); setDrawKind(null); setRenaming(null); setCollapsed(new Set()); setSelection("image-root");
onChange((current) => createEmptyImage(current, template?.id));
}) },
+ { label: "Add image file", icon: <FolderOpen16 />, disabled: !isTauri(), action: () => attempt(async () => {
+ const source = await invoke<ImageSource | null>("open_image_source", { folderPath });
+ if (source) replaceScene(createImageEditScene(source, scene));
+ }) },
...(imageMenu.id ? [
{ label: "Edit", icon: <Edit16 />, action: () => attempt(() => config.assets.find((asset) => asset.id === imageMenu.id)?.imageDraft ? selectImage(imageMenu.id!) : replaceScene(editGeneratedImage(config, imageMenu.id!).imageScene!)) },
{ label: "Remove", icon: <Delete16 />, danger: true, disabled: workItems.some((item) => isWorkActive(item) && item.imageAssetId === imageMenu.id), action: () => removeImage(imageMenu.id!) },
@@ -527,7 +539,7 @@ export function ImageEditor({ config, folderPath, onChange: changeConfig, onGene
onPointerUpCapture={endPan} onPointerCancelCapture={endPan} onLostPointerCapture={endPan}
onAuxClick={(event) => { if (event.button === 1) event.preventDefault(); }}>
<div className="image-frame" style={{ "--image-ratio": width / height, "--image-zoom": view.zoom, "--image-pan-x": `${view.x}px`, "--image-pan-y": `${view.y}px` } as CSSProperties}>
- {imageRoot && scene.sourceImage ? <ReferenceImage folderPath={folderPath} relativePath={scene.sourceImage.relativePath} alt={scene.sourceImage.name} /> : output && !imageRoot && (output.relativePath || output.sourcePath) ? <ReferenceImage folderPath={folderPath} relativePath={output.relativePath} sourcePath={output.sourcePath} alt={output.name} /> : <div className="image-empty"><Image32 aria-hidden="true" /><strong>{imageRoot ? "Open an image to edit" : "Compose your image"}</strong><span>{imageRoot ? "Choose Open image in the inspector, then add Object nodes for edits." : "Add objects, text and groups, then describe them in the inspector."}</span></div>}
+ {imageRoot && scene.sourceImage ? <ReferenceImage folderPath={folderPath} relativePath={scene.sourceImage.relativePath} alt={scene.sourceImage.name} /> : output && !imageRoot && (output.relativePath || output.sourcePath) ? <ReferenceImage folderPath={folderPath} relativePath={output.relativePath} sourcePath={output.sourcePath} alt={output.name} /> : <div className="image-empty"><Image32 aria-hidden="true" /><strong>{imageRoot ? "Add an image to edit" : "Compose your image"}</strong><span>{imageRoot ? "Choose Add image file in the image bar menu, then enter an edit prompt." : "Add objects, text and groups, then describe them in the inspector."}</span></div>}
<svg className={`image-overlay ${drawKind ? "drawing" : ""}`} viewBox="0 0 1000 1000" preserveAspectRatio="none" aria-label="Image placement canvas" onPointerDown={pointerDown} onPointerMove={pointerMove} onPointerUp={pointerUp} onPointerCancel={() => { pointer.current = null; setDraftBox(null); }}>
{boxes && scene.nodes.filter((node) => node.box).map((node) => {
const box = pointer.current?.id === node.id && draftBox ? draftBox : node.box!;
@@ -556,22 +568,21 @@ export function ImageEditor({ config, folderPath, onChange: changeConfig, onGene
data-tooltip="Rename"
onChange={(event) => { if (event.target.value.trim()) patchNode({ name: event.target.value }); }}
/></h2>}
- meta={selected.kind[0].toUpperCase() + selected.kind.slice(1)}
+ meta={imageRoot ? selected.kind === "root" ? "Image" : "Edit" : selected.kind[0].toUpperCase() + selected.kind.slice(1)}
/>
<div className="image-inspector__fields">
- {selected.kind === "root" && <PropRow label="Type" htmlFor={rootField("type")}>
- <ComboBox id={rootField("type")} aria-label="Type" value={imageRoot ? "image" : "prompt"} options={[{ value: "prompt", label: "Prompt" }, { value: "image", label: "Image" }]}
- onChange={(value) => replaceScene(value === "image" ? createImageEditScene(null, scene) : { ...createImageScene(), steps: scene.steps, seed: scene.seed })} />
- </PropRow>}
- {selected.kind === "root" && imageRoot ? <>
- <button className="secondary-button" disabled={!isTauri()} onClick={() => attempt(async () => {
- const source = await invoke<ImageSource | null>("open_image_source", { folderPath });
- if (source) replaceScene(createImageEditScene(source, scene));
- })}><FolderOpen16 aria-hidden="true" />Open image</button>
+ {selected.kind === "root" && imageRoot && <>
{scene.sourceImage && <p className="image-inspector__caption">{scene.sourceImage.name} · {scene.sourceImage.width} × {scene.sourceImage.height}</p>}
- </> : <>
- <label className="image-inspector__area">{selected.kind === "root" ? "Prompt (high-level description)" : "Description"}<textarea className="text-field" rows={4} value={selected.description} onChange={(event) => patchNode({ description: event.target.value })} /></label>
+ <p className="image-inspector__caption">Applies to the whole image before the individual edits.</p>
</>}
+ <label className="image-inspector__area">{imageRoot ? "Edit prompt" : selected.kind === "root" ? "Prompt (high-level description)" : "Description"}<textarea className="text-field" rows={4} value={selected.description} onChange={(event) => patchNode({ description: event.target.value })} /></label>
+ {imageRoot && <PropSection title="References" persistKey="image.edit.references">
+ {config.references.filter((reference) => reference.kind === "image" || reference.kind === "text").map((reference) => {
+ const referenceIds = selected.referenceIds ?? scene.referenceIds;
+ return <label className="image-check" key={reference.id}><input type="checkbox" checked={referenceIds.includes(reference.id)} onChange={(event) => patchNode({ referenceIds: event.target.checked ? [...referenceIds, reference.id] : referenceIds.filter((id) => id !== reference.id) })} />{reference.name}</label>;
+ })}
+ {!config.references.some((reference) => reference.kind === "image" || reference.kind === "text") && <p className="image-inspector__caption">Add image or text references under References, then link them to this edit.</p>}
+ </PropSection>}
{selected.kind === "text" && <PropRow label="Text to render" htmlFor={rootField("text")}><input id={rootField("text")} className="text-field" value={selected.text} onChange={(event) => patchNode({ text: event.target.value })} /></PropRow>}
{selected.kind === "root" && !imageRoot && <>
<label className="image-inspector__area">Background (environment)<textarea className="text-field" rows={3} value={scene.background} onChange={(event) => commit({ ...scene, background: event.target.value })} /></label>
@@ -599,7 +610,7 @@ export function ImageEditor({ config, folderPath, onChange: changeConfig, onGene
<input id={rootField(field)} className="text-field" type="number" min={field === "x" || field === "y" ? 0 : 1} max="1000" value={selected.box![field]} onChange={(event) => { const value = Number(event.target.value); const box = { ...selected.box!, [field]: value }; if (Number.isFinite(value) && box.x >= 0 && box.y >= 0 && box.width > 0 && box.height > 0 && box.x + box.width <= 1000 && box.y + box.height <= 1000) commit(resizeImageNode(scene, selected.id, box)); }} />
</PropRow>)}
</PropSection>}
- {!(selected.kind === "root" && imageRoot) && <PropSection title="Color palette" persistKey="image.palette">
+ {(!imageRoot || selected.colors.length > 0) && <PropSection title="Color palette" persistKey="image.palette">
<div className="image-palette">{selected.colors.map((color, index) => <div key={index}><input aria-label={`Palette color ${index + 1}`} type="color" value={color} onChange={(event) => patchNode({ colors: selected.colors.map((old, i) => i === index ? event.target.value : old) })} /><button className="icon-button image-pane-button" aria-label={`Remove color ${index + 1}`} data-tooltip="Remove color" onClick={() => patchNode({ colors: selected.colors.filter((_, i) => i !== index) })}><Delete14 aria-hidden="true" /></button></div>)}<button className="secondary-button" disabled={selected.colors.length >= (selected.kind === "root" ? 16 : 5)} onClick={() => patchNode({ colors: [...selected.colors, "#808080"] })}><Add14 aria-hidden="true" />Color</button></div>
</PropSection>}
</div>
diff --git a/src/lib/imageCommands.ts b/src/lib/imageCommands.ts
index 7c37889..afd893d 100644
--- a/src/lib/imageCommands.ts
+++ b/src/lib/imageCommands.ts
@@ -3,7 +3,7 @@ import { imageDescendants, imageNodeSchema, imageSceneSchema, removeImageNode, r
const id = z.string().min(1);
const before = id.nullish();
-const editableNode = imageNodeSchema.pick({ name: true, description: true, text: true, box: true, colors: true }).partial();
+const editableNode = imageNodeSchema.pick({ name: true, description: true, text: true, box: true, colors: true, referenceIds: true }).partial();
const authoredScene = imageSceneSchema.innerType().omit({ outputAssetId: true, rootType: true, sourceImage: true });
const configuration = authoredScene.omit({ nodes: true, style: true }).partial().extend({ prompt: z.string().optional(), style: authoredScene.shape.style.partial().strict().optional() });
export const imageCommandSchema = z.discriminatedUnion("op", [
@@ -52,7 +52,7 @@ export function applyImageCommand(current: ImageScene, command: ImageCommand): I
break;
}
case "image.node.add":
- insert([{ id: command.id, parentId: command.parent, kind: command.kind, name: command.name, description: command.description ?? "", text: command.text ?? "", box: command.box ?? null, colors: command.colors ?? [] }], command.parent, command.before);
+ insert([{ id: command.id, parentId: command.parent, kind: command.kind, name: command.name, description: command.description ?? "", text: command.text ?? "", box: command.box ?? null, colors: command.colors ?? [], ...(command.referenceIds ? { referenceIds: command.referenceIds } : {}) }], command.parent, command.before);
break;
case "image.node.set": {
node(command.id);
diff --git a/src/lib/imageEditing.test.ts b/src/lib/imageEditing.test.ts
index 22cd1c8..b582bec 100644
--- a/src/lib/imageEditing.test.ts
+++ b/src/lib/imageEditing.test.ts
@@ -56,9 +56,35 @@ it("starts with a clean image root and requires described edits before submissio
expect(edited.imageScene).toMatchObject({ rootType: "image", sourceImage: { width: 101, height: 77 }, outputAssetId: null });
expect(edited.imageScene!.nodes).toHaveLength(1);
expect(edited.imageScene!.nodes[0].description).toBe("");
- expect(() => compileImageEdits(edited)).toThrow("Add an Object");
+ expect(() => compileImageEdits(edited)).toThrow("Enter a whole-image edit prompt");
edited.imageScene = addImageNode(edited.imageScene!, "image-root", "object");
expect(() => compileImageEdits(edited)).toThrow("Describe the edit");
expect(config.imageScene!.nodes.length).toBeGreaterThan(1);
expect(parseProjectConfig(JSON.parse(JSON.stringify(edited))).imageScene).toEqual(edited.imageScene);
});
+
+it("applies whole-image actions first and compiles only each edit's linked references", () => {
+ const config = parseProjectConfig(fixture);
+ config.imageScene = createImageEditScene({ relativePath: "media/source.png", name: "Source", width: 101, height: 77 });
+ config.references = ["Lighting", "Balloon", "Unused"].map((name) => ({ id: name, name, kind: "image", description: name, relativePath: `references/${name}.png`, intendedUse: [], createdAt: config.createdAt }));
+ const root = config.imageScene.nodes[0];
+ root.description = "Warm up the entire image";
+ root.referenceIds = ["Lighting"];
+ expect(compileImageEdits(config).edits).toHaveLength(1);
+ config.imageScene = addImageNode(config.imageScene, root.id, "object");
+ Object.assign(config.imageScene.nodes[1], { description: "Add a balloon", referenceIds: ["Balloon"] });
+ let edits = compileImageEdits(config).edits;
+ expect(edits[0]).toMatchObject({ x: 0, y: 0, width: 101, height: 77 });
+ expect(edits[0].prompt).toContain("Warm up the entire image");
+ expect(edits[0].prompt).toContain("<Subject 1> is Lighting, providing appearance from <Picture 2>");
+ expect(edits[0].prompt).not.toContain("Balloon");
+ expect(edits[1].prompt).toContain("<Subject 1> is Balloon, providing appearance from <Picture 2>");
+ expect(edits[1].prompt).not.toContain("Lighting");
+ expect(edits.map((edit) => edit.references.map((reference) => reference.id))).toEqual([["Lighting"], ["Balloon"]]);
+ config.imageScene.referenceIds = ["Unused"];
+ config.imageScene.nodes[1].referenceIds = [];
+ edits = compileImageEdits(config).edits;
+ expect(edits[1].references).toEqual([]);
+ config.imageScene.nodes[1].referenceIds = ["missing"];
+ expect(() => compileImageEdits(config)).toThrow("no longer exists");
+});
diff --git a/src/lib/imageEditing.ts b/src/lib/imageEditing.ts
index d4bd60b..5b7c912 100644
--- a/src/lib/imageEditing.ts
+++ b/src/lib/imageEditing.ts
@@ -1,16 +1,22 @@
import { createImageEditScene, createImageScene, imageSourceSchema, type ImageNode, type ImageSource } from "./imageScene";
import { compileImagePrompt } from "./imagePrompt";
-import type { ProjectConfig } from "./project";
+import type { ProjectConfig, ProjectReference } from "./project";
-export interface ImageEditStep { prompt: string; x: number; y: number; width: number; height: number }
+export interface ImageEditStep {
+ prompt: string; x: number; y: number; width: number; height: number;
+ referencePaths?: string[];
+ refmods?: Array<{ path: string; strength: number; copies: number }>;
+}
export const imageEditDebugPrompt = (edits: ImageEditStep[]) => edits.map((edit, index) => `Edit ${index + 1} (${edit.x}, ${edit.y}, ${edit.width}, ${edit.height})\n${edit.prompt}`).join("\n\n");
-export function compileImageEdits(config: ProjectConfig): { source: ImageSource; edits: ImageEditStep[] } {
+export function compileImageEdits(config: ProjectConfig): { source: ImageSource; edits: (ImageEditStep & { references: ProjectReference[] })[] } {
const scene = config.imageScene!;
const source = scene.sourceImage;
if (!source) throw new Error("Open an image before adding edits.");
- const ordered: ImageNode[] = [];
+ const root = scene.nodes.find((node) => node.kind === "root")!;
+ // The root describes a whole-image edit, followed by the local edits in order.
+ const ordered: ImageNode[] = root.description.trim() ? [{ ...root, box: { x: 0, y: 0, width: 1000, height: 1000 } }] : [];
const visit = (parent: string) => { for (const node of scene.nodes.filter((item) => item.parentId === parent)) { ordered.push(node); visit(node.id); } };
- visit(scene.nodes.find((node) => node.kind === "root")!.id);
+ visit(root.id);
const edits = ordered.filter((node) => node.kind !== "group" || node.description.trim() || node.text || node.colors.length).map((node) => {
if (!node.box) throw new Error(`Set a placement box for '${node.name}'.`);
if (!node.description.trim() && !node.text && !node.colors.length) throw new Error(`Describe the edit for '${node.name}'.`);
@@ -20,11 +26,11 @@ export function compileImageEdits(config: ProjectConfig): { source: ImageSource;
const bottom = Math.min(source.height, Math.ceil((node.box.y + node.box.height) * source.height / 1000));
const description = ["The edited keyframe corresponds to <Picture 1> with the following change. Preserve the composition and any previously completed edits.", node.description,
node.text ? `Render the exact text "${node.text}".` : "", node.colors.length ? `Use these colors: ${node.colors.join(", ")}.` : ""].filter(Boolean).join(" ");
- const editScene = { ...createImageScene(description), rootType: "image" as const, sourceImage: source, referenceIds: scene.referenceIds };
+ const editScene = { ...createImageScene(description), rootType: "image" as const, sourceImage: source, referenceIds: node.referenceIds ?? scene.referenceIds };
const compiled = compileImagePrompt({ ...config, settings: { ...config.settings, defaultLook: null }, imageScene: editScene });
- return { prompt: compiled.prompt, x, y, width: Math.max(1, right - x), height: Math.max(1, bottom - y) };
+ return { prompt: compiled.prompt, references: compiled.references, x, y, width: Math.max(1, right - x), height: Math.max(1, bottom - y) };
});
- if (!edits.length) throw new Error("Add an Object node and describe an edit before generating.");
+ if (!edits.length) throw new Error("Enter a whole-image edit prompt or add an Edit node before generating.");
return { source, edits };
}
diff --git a/src/lib/imageHistory.test.ts b/src/lib/imageHistory.test.ts
index 225d2c8..6d95520 100644
--- a/src/lib/imageHistory.test.ts
+++ b/src/lib/imageHistory.test.ts
@@ -1,11 +1,30 @@
import { describe, expect, it } from "vitest";
-import { imageGenerationSnapshot, restoreGeneratedImage } from "./imageHistory";
+import { imageGenerationSnapshot, restoreGeneratedImage, saveImageDraft } from "./imageHistory";
import { imageGenerationSnapshotSchema, parseProjectConfig } from "./project";
import fixture from "../../fixtures/project-v1-image.json";
import snapshotFixture from "../../fixtures/image-generation-snapshot.json";
import { createImageEditScene } from "./imageScene";
describe("generated image history", () => {
+ it("preserves a whole-image prompt and linked references as a new draft of a saved image", () => {
+ const config = parseProjectConfig(fixture);
+ const source = { relativePath: "media/generated/source.png", name: "Source", width: 101, height: 77 };
+ config.assets = [{ id: "saved", kind: "image", ...source, mimeType: "image/png", createdAt: config.createdAt }];
+ config.imageScene = { ...createImageEditScene(source), outputAssetId: "saved" };
+ config.imageScene.nodes[0].description = "Make the lighting warmer";
+ config.imageScene.nodes[0].referenceIds = ["mood"];
+ config.references = [{ id: "mood", name: "Mood", kind: "text", description: "Warm sunset", intendedUse: [], createdAt: config.createdAt }];
+ const saved = saveImageDraft(config, "test");
+ expect(saved.assets).toHaveLength(2);
+ expect(saved.imageScene!.outputAssetId).not.toBe("saved");
+ const reopened = parseProjectConfig(JSON.parse(JSON.stringify(saved)));
+ reopened.references = [];
+ const restored = restoreGeneratedImage(reopened, saved.imageScene!.outputAssetId!);
+ expect(restored.references).toEqual(config.references);
+ expect(restored.imageScene!.nodes[0]).toMatchObject({ description: "Make the lighting warmer", referenceIds: ["mood"] });
+ const snapshot = reopened.assets[1].imageGeneration!;
+ expect(imageGenerationSnapshotSchema.safeParse({ ...snapshot, references: [] }).success).toBe(false);
+ });
it("restores saved inputs and references after reopening without changing other results", () => {
const config = parseProjectConfig(fixture);
const snapshot = imageGenerationSnapshotSchema.parse(snapshotFixture);
diff --git a/src/lib/imageHistory.ts b/src/lib/imageHistory.ts
index a992c79..786799c 100644
--- a/src/lib/imageHistory.ts
+++ b/src/lib/imageHistory.ts
@@ -1,4 +1,4 @@
-import { createImageEditScene, createImageScene } from "./imageScene";
+import { createImageEditScene, createImageScene, imageSceneReferenceIds } from "./imageScene";
import type { ImageGenerationSnapshot, ProjectAsset, ProjectConfig } from "./project";
export function imageGenerationSnapshot(config: ProjectConfig, prompt: string, generatorTemplateId: string): ImageGenerationSnapshot {
@@ -11,7 +11,7 @@ export function imageGenerationSnapshot(config: ProjectConfig, prompt: string, g
briefPrompt: config.brief.prompt,
prompt,
generatorTemplateId,
- references: scene.referenceIds.map((id) => config.references.find((reference) => reference.id === id)!),
+ references: imageSceneReferenceIds(scene).map((id) => config.references.find((reference) => reference.id === id)!),
});
}
@@ -22,7 +22,7 @@ export function saveImageDraft(config: ProjectConfig, generatorTemplateId?: stri
const selected = config.assets.find((asset) => asset.id === scene.outputAssetId);
const source = scene.rootType === "image" ? scene.sourceImage : null;
if (!source && !selected?.imageDraft) return config;
- if (!selected?.imageDraft && scene.outputAssetId && scene.nodes.length === 1) return config;
+ if (!selected?.imageDraft && scene.outputAssetId && scene.nodes.length === 1 && !scene.nodes[0].description.trim()) return config;
const existing = selected?.imageDraft ? selected : undefined;
const id = existing?.id ?? `image-draft-${crypto.randomUUID()}`;
const snapshot = imageGenerationSnapshot(config, "", generatorTemplateId || existing?.imageGeneration?.generatorTemplateId || selected?.imageGeneration?.generatorTemplateId || "image-draft");
diff --git a/src/lib/imageScene.ts b/src/lib/imageScene.ts
index 794d3c0..3bc7b0f 100644
--- a/src/lib/imageScene.ts
+++ b/src/lib/imageScene.ts
@@ -6,6 +6,7 @@ export const imageNodeSchema = z.object({
id: z.string().min(1), parentId: z.string().min(1).nullable(),
kind: z.enum(["root", "group", "object", "text", "background"]),
name: z.string().min(1).max(120), description: z.string(), text: z.string(),
+ referenceIds: z.array(z.string().min(1)).max(100).optional(),
box: boxSchema.nullable(), colors: z.array(z.string().regex(/^#[0-9a-fA-F]{6}$/)).max(16),
});
export const imageSourceSchema = z.object({
@@ -41,6 +42,9 @@ export type ImageScene = z.infer<typeof imageSceneSchema>;
export type ImageNode = z.infer<typeof imageNodeSchema>;
export type ImageBox = NonNullable<ImageNode["box"]>;
+export const imageSceneReferenceIds = (scene: ImageScene): string[] =>
+ [...new Set([...scene.referenceIds, ...scene.nodes.flatMap((node) => node.referenceIds ?? [])])];
+
export function createImageScene(prompt = ""): ImageScene {
return { nodes: [{ id: "image-root", parentId: null, kind: "root", name: "Image", description: prompt, text: "", box: null, colors: [] }],
background: "", style: { mode: "photo", aesthetics: "", lighting: "", medium: "", detail: "" }, steps: 20, seed: -1, referenceIds: [], outputAssetId: null };
diff --git a/src/lib/project.ts b/src/lib/project.ts
index 4ba4e3d..8798298 100644
--- a/src/lib/project.ts
+++ b/src/lib/project.ts
@@ -1,5 +1,5 @@
import { z } from "zod";
-import { createImageScene, imageSceneSchema } from "./imageScene";
+import { createImageScene, imageSceneReferenceIds, imageSceneSchema } from "./imageScene";
import { effectEnabledSchema, effectSchemas, isEffectOn } from "./effectSettings";
import { normalizeShotTagSelection, shotTagClauses, SHOT_TAG_ID_PATTERN, type ShotTagClauses, type ShotTagSelection } from "./shot-tags";
@@ -152,7 +152,7 @@ export const imageGenerationSnapshotSchema = z.object({
references: z.array(z.lazy(() => projectReferenceSchema)).max(100),
}).superRefine((snapshot, context) => {
const ids = new Set(snapshot.references.map((reference) => reference.id));
- if (ids.size !== snapshot.references.length || snapshot.scene.referenceIds.some((id) => !ids.has(id))) {
+ if (ids.size !== snapshot.references.length || imageSceneReferenceIds(snapshot.scene).some((id) => !ids.has(id))) {
context.addIssue({ code: z.ZodIssueCode.custom, message: "Image generation references must have unique IDs and include every selected reference." });
}
});
diff --git a/src/lib/workQueue.test.ts b/src/lib/workQueue.test.ts
index 1391cc5..e2035c5 100644
--- a/src/lib/workQueue.test.ts
+++ b/src/lib/workQueue.test.ts
@@ -85,6 +85,40 @@ describe("image generation work", () => {
expect(result.thumbnail).toBe("media/third.jpg");
expect(result.assets.filter((asset) => [first, second].some((item) => asset.relativePath === `media/${item.id}.jpg`))).toHaveLength(2);
});
+ it("submits whole-image and local edits with separate reference pictures and refmods", async () => {
+ const { queue } = setup();
+ const record = imageProject();
+ let scene = createImageEditScene({ name: "Source", relativePath: "media/generated/source.jpg", width: 101, height: 77 });
+ scene.nodes[0].description = "Make the light warmer";
+ scene.nodes[0].referenceIds = ["Lighting"];
+ scene = addImageNode(scene, "image-root", "object");
+ Object.assign(scene.nodes[1], { name: "Edit", description: "Add a red balloon", referenceIds: ["Balloon"] });
+ record.config.imageScene = scene;
+ record.config.references = ["Lighting", "Balloon"].map((name) => ({ id: name, name, kind: "image", description: name,
+ relativePath: `references/${name}.png`, intendedUse: [], createdAt: record.config.createdAt,
+ refmods: [{ id: name, name, relativePath: `references/${name}.safetensors`, strength: 0.75, copies: 2 }] }));
+ const session = queue.project(record);
+ vi.mocked(invoke).mockResolvedValue({ relativePath: "media/generated/edited.png", width: 101, height: 77 });
+ queue.enqueueImage(session, template);
+ await waitFor(() => expect(enqueueSlopfabGeneration).toHaveBeenCalledOnce());
+ const request = vi.mocked(enqueueSlopfabGeneration).mock.calls[0][0];
+ const steps = request.imageEdit!.edits;
+ expect(steps).toHaveLength(2);
+ expect(steps[0]).toMatchObject({ x: 0, y: 0, width: 101, height: 77 });
+ for (const [index, name] of ["Lighting", "Balloon"].entries()) {
+ expect(steps[index].referencePaths).toEqual(["C:/Image/media/generated/source.jpg", `C:/Image/references/${name}.png`]);
+ expect(steps[index].refmods).toEqual([{ path: `C:/Image/references/${name}.safetensors`, strength: 0.75, copies: 2 }]);
+ expect(steps[index].prompt).toContain(`<Subject 1> is ${name}, providing appearance from <Picture 2>`);
+ }
+ expect(request.referencePaths).toEqual(steps[0].referencePaths);
+ expect(request.refmods).toEqual(steps[0].refmods);
+ expect(vi.mocked(resolveSlopfabPlan).mock.calls[0][0].imageEdit).toEqual(request.imageEdit);
+ await finish(queue, request.jobId);
+ const result = parseProjectConfig(JSON.parse(JSON.stringify(session.getSnapshot().config)));
+ expect(result.assets[0].imageGeneration!.references.map((reference) => reference.id)).toEqual(["Lighting", "Balloon"]);
+ expect(result.imageScene!.nodes).toHaveLength(1);
+ expect(result.imageScene!.nodes[0].description).toBe("");
+ });
it("submits ordered inpainting as one job, saves PNG, and leaves a clean edit root", async () => {
const { queue } = setup();
const record = imageProject();
diff --git a/src/lib/workQueue.ts b/src/lib/workQueue.ts
index 333c11a..b585020 100644
--- a/src/lib/workQueue.ts
+++ b/src/lib/workQueue.ts
@@ -214,17 +214,22 @@ export class WorkQueue {
if (this.items.some((item) => item.projectKey === projectKey && item.kind === "image" && item.imageAssetId === scene.outputAssetId && isWorkActive(item))) return;
const config = structuredClone({ ...current, providerSettings: { ...current.providerSettings,
slopfab: engineProviderSetting(template.paths, current.providerSettings.slopfab, template.attention, template.loras, "prompt", template.additionalSafetensors, false) } });
- const { prompt, references } = compileImagePrompt(current);
+ const { prompt, references } = edit ? edit.edits[0] : compileImagePrompt(current);
+ const referencePaths = (selected: typeof references) => [
+ ...(edit ? [projectItemPath(session.record.folderPath, edit.source)!] : []),
+ ...selected.flatMap((reference) => referenceImages(reference).map((image) => projectItemPath(session.record.folderPath, image)!)),
+ ];
const { width, height } = edit?.source ?? outputDimensions(current.settings.resolution, current.settings.aspectRatio);
const id = `image-${crypto.randomUUID()}`;
const imageDraftId = current.assets.find((asset) => asset.id === scene.outputAssetId && asset.imageDraft)?.id;
let finish!: () => void;
const done = new Promise<void>((resolve) => { finish = resolve; });
const request: SlopfabGenerationRequest = { jobId: id, stillImage: true, frames: 1, prompt: edit ? edit.edits[0].prompt : prompt,
- ...(edit ? { imageEdit: { sourceRelativePath: edit.source.relativePath, edits: edit.edits } } : {}),
+ ...(edit ? { imageEdit: { sourceRelativePath: edit.source.relativePath, edits: edit.edits.map(({ references, ...step }) => ({
+ ...step, referencePaths: referencePaths(references), refmods: referenceRefmodInputs(session.record.folderPath, references),
+ })) } } : {}),
canvasWidth: width, canvasHeight: height, steps: generationStepsWithLoras(scene.steps, config), seed: scene.seed,
- referencePaths: [...(edit ? [projectItemPath(session.record.folderPath, edit.source)!] : []),
- ...references.flatMap((reference) => referenceImages(reference).map((image) => projectItemPath(session.record.folderPath, image)!))],
+ referencePaths: referencePaths(references),
refmods: referenceRefmodInputs(session.record.folderPath, references),
};
this.work.set(id, { id, image: true, imageDraftId, imageGeneration: imageGenerationSnapshot(config, edit ? imageEditDebugPrompt(edit.edits) : prompt, template.id), session, sceneId: scene.nodes.find((node) => node.kind === "root")!.id, config, snapshot: JSON.stringify(scene), request, submitted: false, cancelled: false, done, finish });