Skip to content

AI media harness for planning, generating, and editing locally

TypeScript

Create scenes and references from images

Bitti2d66f9248fe3
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 0665483..45f27da 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -4,6 +4,7 @@ This alpha update adds DMAD LoRA support, resumable model downloads, and improve
## What's new
+- Create a video scene with an image as its first frame, or create an image reference, directly from the Image tab's image bar context menu.
- Every project now includes Video and Image tabs. Older video and image projects keep their saved content, and Export supports both output types.
- Clothes and Accessories reference categories include common wearable items. Clothes offer optional color and fabric settings.
- Video references now offer a Frames mode: select still frames from a clip and use them as image references.
diff --git a/src/components/ImageProject.integration.test.tsx b/src/components/ImageProject.integration.test.tsx
index 0ff3437..c6550bc 100644
--- a/src/components/ImageProject.integration.test.tsx
+++ b/src/components/ImageProject.integration.test.tsx
@@ -23,6 +23,30 @@ vi.mock("@tauri-apps/api/event", () => ({ listen: vi.fn(async () => () => undefi
afterEach(() => { cleanup(); localStorage.clear(); vi.restoreAllMocks(); });
const record = (): ProjectRecord => ({ folderPath: "D:/Images", config: parseProjectConfig(fixture) });
+it.each(["Create Scene", "Create Reference"])("opens and saves the item created by %s in the image bar", async (action) => {
+ const project = record();
+ project.config.assets = [{ id: "poster", name: "Poster", kind: "image", relativePath: "media/generated/poster.png", mimeType: "image/png", createdAt: project.config.createdAt }];
+ project.config.imageScene!.outputAssetId = "poster";
+ const save = vi.fn(async (_record: ProjectRecord) => undefined);
+ render(<ProjectWorkspace project={project} onBack={vi.fn()} onSave={save} />);
+ fireEvent.contextMenu(screen.getByRole("button", { name: "View Poster" }));
+ fireEvent.click(screen.getByRole("menuitem", { name: action }));
+ const navigation = within(screen.getByRole("tablist", { name: "Project views" }));
+ expect(navigation.getByRole("tab", { name: action === "Create Scene" ? "Video" : "References" })).toHaveAttribute("aria-selected", "true");
+ if (action === "Create Scene") expect(screen.getByRole("combobox", { name: "Start frame for this scene" })).toHaveTextContent("Poster");
+ else expect(screen.getByLabelText("Reference name")).toHaveValue("Poster");
+ fireEvent.click(screen.getByRole("button", { name: "Save" }));
+ await waitFor(() => expect(save).toHaveBeenCalledOnce());
+ const saved = parseProjectConfig(JSON.parse(JSON.stringify(save.mock.calls[0][0].config)));
+ expect(saved.references[0].images?.[0].relativePath).toBe("media/generated/poster.png");
+ if (action === "Create Scene") expect(saved.generationJobs.at(-1)?.startFrameReferenceId).toBe(saved.references[0].id);
+ fireEvent.click(screen.getByRole("button", { name: "Undo" }));
+ fireEvent.click(screen.getByRole("button", { name: "Save" }));
+ await waitFor(() => expect(save).toHaveBeenCalledTimes(2));
+ expect(save.mock.calls[1][0].config.references).toEqual(project.config.references);
+ expect(save.mock.calls[1][0].config.generationJobs).toEqual(project.config.generationJobs);
+});
+
it.each([false, true])("scopes generation controls to each image and queues others (draft: %s)", async (imageDraft) => {
vi.spyOn(persistence, "isTauri").mockReturnValue(true);
vi.spyOn(persistence, "readMediaFileUrl").mockResolvedValue(null);
diff --git a/src/components/ProjectWorkspace.tsx b/src/components/ProjectWorkspace.tsx
index c0e5d2b..08b8f43 100644
--- a/src/components/ProjectWorkspace.tsx
+++ b/src/components/ProjectWorkspace.tsx
@@ -286,7 +286,7 @@ export function ProjectWorkspace({ project, initialView, runtime = null, onBack,
<ProjectStatusSlot.Provider value={statusSlot}>
<div className={`project-body${agentOpen ? " project-body--agent" : ""}`} style={agentPane.style}>
<div id={`${panelId}-view`} role="tabpanel" aria-label={VIEW_LABELS[view]} className={`project-content project-content--${view}`}>
- {view === "editor" && <ImageEditor config={config} folderPath={project.folderPath} onChange={changeConfig} onGenerate={(template) => queue.enqueueImage(session, template)} onCancel={(id) => queue.cancel(id)} workItems={projectItems.filter((item) => item.kind === "image")} />}
+ {view === "editor" && <ImageEditor config={config} folderPath={project.folderPath} onChange={changeConfig} onGenerate={(template) => queue.enqueueImage(session, template)} onCancel={(id) => queue.cancel(id)} workItems={projectItems.filter((item) => item.kind === "image")} onOpenGenerator={(jobId) => { setSelectedGenerationJobId(jobId); setView("generator"); }} onOpenReferences={() => setView("references")} />}
{view === "timeline" && <TimelineView config={config} openSceneId={selectedGenerationJobId} folderPath={project.folderPath} generationCompletionTimes={generationCompletionTimes} onChange={changeConfig} onMeasured={recordMeasurement} onOpenGenerator={(jobId) => { setSelectedGenerationJobId(jobId); setView("generator"); }} />}
{view === "generator" && <GeneratorView onGenerate={(submissions) => queue.enqueue(session, submissions)} onCancelGeneration={(ids) => queue.cancelScenes(session, ids)} cancellingJobIds={cancellingJobIds} config={config} folderPath={project.folderPath} generationCompletionTimes={generationCompletionTimes} runtime={runtime?.slopfab ?? null} onRuntimeChange={onGeneratorRuntimeChange} onChange={changeConfig} selectedJobId={selectedGenerationJobId} onSelectedJobChange={setSelectedGenerationJobId} onOpenTimeline={() => setView("timeline")} />}
{view === "references" && <ReferencesView config={config} folderPath={project.folderPath} onChange={changeConfig} onRegenerateIcon={(id) => queue.regenerateReferenceIcon(session, id)} onGenerateBuiltinIcons={() => queue.generateBuiltinReferenceIcons(session)} onRegenerateBuiltinIcon={(id) => queue.regenerateBuiltinReferenceIcon(session, id)} pendingBuiltinIconIds={queue.pendingBuiltinIconIds()} pendingIconIds={new Set(config.references.filter((reference) => queue.isReferenceIconPending(session, reference.id)).map((reference) => reference.id))} onOpenGenerator={(jobId) => { setSelectedGenerationJobId(jobId); setView("generator"); }} />}
diff --git a/src/components/workspace/ImageEditor.test.tsx b/src/components/workspace/ImageEditor.test.tsx
index 1d93cc4..2695cd5 100644
--- a/src/components/workspace/ImageEditor.test.tsx
+++ b/src/components/workspace/ImageEditor.test.tsx
@@ -6,7 +6,7 @@ import { promptValue, typePrompt } from "./promptTestUtils";
import { afterEach, expect, it, vi } from "vitest";
import { ImageEditor } from "./ImageEditor";
import { choose, comboValue } from "./comboTestUtils";
-import { imageGenerationSnapshotSchema, parseProjectConfig, type ProjectConfig } from "../../lib/project";
+import { imageGenerationSnapshotSchema, parseProjectConfig, referenceImages, sceneFrameInputs, type ProjectConfig } from "../../lib/project";
import { imageGenerationSnapshot } from "../../lib/imageHistory";
import { createGeneratorTemplate, defaultGeneratorTemplate, saveGeneratorTemplateSettings } from "../../lib/settings";
import { createImageEditScene, imageScenePrompt } from "../../lib/imageScene";
@@ -32,6 +32,53 @@ function setup(initial = parseProjectConfig(fixture)) {
/* Undo lives in the title bar; inside the editor it is Ctrl+Z. */
const undoImageEdit = () => fireEvent.keyDown(screen.getByRole("tree", { name: "Image nodes" }), { key: "z", code: "KeyZ", ctrlKey: true });
+it.each(["Create Scene", "Create Reference"])("%s uses the right-clicked image without changing the viewed image", (action) => {
+ const initial = parseProjectConfig(fixture);
+ initial.assets = ["Viewed", "Chosen"].map((name) => ({ id: name, name, kind: "image", relativePath: `media/generated/${name}.png`, mimeType: "image/png", createdAt: initial.createdAt }));
+ initial.imageScene!.outputAssetId = "Viewed";
+ const current = setup(initial);
+ fireEvent.contextMenu(screen.getByRole("button", { name: "View Chosen" }));
+ fireEvent.click(screen.getByRole("menuitem", { name: action }));
+ const reopened = parseProjectConfig(JSON.parse(JSON.stringify(current())));
+ expect(reopened.references).toHaveLength(initial.references.length + 1);
+ const reference = reopened.references[0];
+ expect(reference.name).toBe("Chosen");
+ expect(referenceImages(reference)[0].relativePath).toBe("media/generated/Chosen.png");
+ expect(reopened.imageScene).toEqual(initial.imageScene);
+ expect(reopened.assets).toEqual(initial.assets);
+ expect(reopened.timeline).toEqual(initial.timeline);
+ if (action === "Create Scene") {
+ const job = reopened.generationJobs.at(-1)!;
+ expect(job).toMatchObject({ title: "Chosen", status: "draft", sceneType: "first-last-frame", startFrameReferenceId: reference.id });
+ expect(sceneFrameInputs(job, reopened).references.map((item) => item.id)).toContain(reference.id);
+ } else expect(reopened.generationJobs).toEqual(initial.generationJobs);
+});
+
+it("creates a reference from an external image variant using the keyboard menu", () => {
+ const initial = parseProjectConfig(fixture);
+ initial.assets = [
+ { id: "original", name: "Original", kind: "image", relativePath: "media/original.png", mimeType: "image/png", createdAt: initial.createdAt },
+ { id: "variant", name: "Variant", parentAssetId: "original", kind: "image", sourcePath: "D:/Pictures/variant.png", mimeType: "image/png", createdAt: initial.createdAt },
+ ];
+ initial.imageScene!.outputAssetId = "original";
+ const current = setup(initial);
+ fireEvent.keyDown(screen.getByRole("button", { name: "View Variant" }), { key: "F10", shiftKey: true });
+ fireEvent.click(screen.getByRole("menuitem", { name: "Create Reference" }));
+ const reopened = parseProjectConfig(JSON.parse(JSON.stringify(current())));
+ expect(referenceImages(reopened.references[0])[0]).toMatchObject({ name: "Variant", sourcePath: "D:/Pictures/variant.png" });
+});
+
+it("disables image reuse for empty drafts and omits it on the empty bar", () => {
+ setup();
+ fireEvent.contextMenu(screen.getByLabelText("Generated images"));
+ expect(screen.queryByRole("menuitem", { name: "Create Scene" })).not.toBeInTheDocument();
+ expect(screen.queryByRole("menuitem", { name: "Create Reference" })).not.toBeInTheDocument();
+ fireEvent.click(screen.getByRole("menuitem", { name: "New empty image" }));
+ fireEvent.contextMenu(screen.getByRole("button", { name: "View New image" }));
+ expect(screen.getByRole("menuitem", { name: "Create Scene" })).toBeDisabled();
+ expect(screen.getByRole("menuitem", { name: "Create Reference" })).toBeDisabled();
+});
+
it.each([false, true])("shows and copies the selected result's exact seed as read-only (image edits: %s)", async (imageRoot) => {
const writeText = vi.fn().mockResolvedValue(undefined);
vi.stubGlobal("navigator", Object.assign(Object.create(navigator), { clipboard: { writeText } }));
diff --git a/src/components/workspace/ImageEditor.tsx b/src/components/workspace/ImageEditor.tsx
index 8fac53a..ee0fcbe 100644
--- a/src/components/workspace/ImageEditor.tsx
+++ b/src/components/workspace/ImageEditor.tsx
@@ -1,5 +1,5 @@
import { useEffect, useId, useMemo, useRef, useState, useSyncExternalStore, type CSSProperties, type DragEvent, type KeyboardEvent as ReactKeyboardEvent, type PointerEvent } from "react";
-import { Add14, Add16, Boxes16, Boxes16Filled, ChevronDown14, ChevronRight14, Copy16, Cube16, Cube16Filled, Cursor16, Cursor16Filled, Delete14, Delete16, Edit16, FolderAdd16, FolderOpen16, Group16, Group16Filled, Image16, Image32, Paste16, Rename16, Sparkle16, Stop14, Text16, Text16Filled } from "../ui/icons";
+import { Add14, Add16, Boxes16, Boxes16Filled, ChevronDown14, ChevronRight14, Copy16, Cube16, Cube16Filled, Cursor16, Cursor16Filled, Delete14, Delete16, Edit16, Film16, FolderAdd16, FolderOpen16, Group16, Group16Filled, Image16, Image32, Paste16, References16, Rename16, Sparkle16, Stop14, Text16, Text16Filled } from "../ui/icons";
import { invoke } from "@tauri-apps/api/core";
import { addImageNode, createImageEditScene, createImageScene, duplicateImageNode, imageDescendants, imageScenePrompt, removeImageNode, resizeImageNode, type ImageBox, type ImageNode, type ImageScene, type ImageSource } from "../../lib/imageScene";
import { compileImageEdits, editGeneratedImage, imageEditDebugPrompt } from "../../lib/imageEditing";
@@ -7,7 +7,7 @@ import { outputDimensions } from "../../lib/export";
import { compileImagePrompt } from "../../lib/imagePrompt";
import { createEmptyImage, imageFamily, removeImageAsset, restoreGeneratedImage, saveImageDraft } from "../../lib/imageHistory";
import { isTauri } from "../../lib/persistence";
-import { isVideoReference, PROJECT_RESOLUTIONS, referenceTypeLabel, type ProjectConfig } from "../../lib/project";
+import { createDraftGenerationJob, isVideoReference, PROJECT_RESOLUTIONS, referenceTypeLabel, type ProjectConfig, type ProjectReference } from "../../lib/project";
import { defaultGeneratorTemplate, loadDebugOptionsEnabled, loadGeneratorTemplateSettings, MINIMAX_H3_MODEL_TYPE, subscribeDebugOptions, subscribeGeneratorTemplates, templateUsable, type GeneratorTemplate } from "../../lib/settings";
import { isWorkActive, type WorkItem } from "../../lib/workQueue";
import type { ConfigUpdate } from "./TimelineView";
@@ -38,9 +38,11 @@ function UsedImageSeed({ id, seed, onError }: { id: string; seed: number | undef
</PropRow>;
}
-export function ImageEditor({ config, folderPath, onChange: changeConfig, onGenerate, onCancel, workItems = [] }: {
+export function ImageEditor({ config, folderPath, onChange: changeConfig, onGenerate, onCancel, onOpenGenerator, onOpenReferences, workItems = [] }: {
config: ProjectConfig; folderPath: string; onChange: (update: ConfigUpdate) => void;
onGenerate: (template: GeneratorTemplate) => void; onCancel: (id: string) => Promise<void>; workItems?: readonly WorkItem[];
+ onOpenGenerator?: (jobId: string) => void;
+ onOpenReferences?: () => void;
}) {
const scene = useMemo(() => config.imageScene ?? createImageScene(config.brief.prompt), [config.imageScene, config.brief.prompt]);
const root = scene.nodes.find((node) => node.kind === "root")!;
@@ -136,6 +138,8 @@ export function ImageEditor({ config, folderPath, onChange: changeConfig, onGene
const active = work && isWorkActive(work);
const output = config.assets.find((asset) => asset.id === scene.outputAssetId);
const images = config.assets.filter((asset) => asset.kind === "image");
+ const menuImage = images.find((asset) => asset.id === imageMenu?.id);
+ const canUseMenuImage = Boolean(menuImage?.relativePath || menuImage?.sourcePath);
// Generated assets retain the dimensions returned by the native encoder.
// Project settings size only the empty canvas and future generation requests.
const { width, height } = imageRoot && scene.sourceImage ? scene.sourceImage : output?.width && output?.height
@@ -191,6 +195,23 @@ export function ImageEditor({ config, folderPath, onChange: changeConfig, onGene
resetImageEditing(id);
onChange((current) => restoreGeneratedImage(current, id));
};
+ const createFromImage = (id: string, asScene: boolean) => {
+ const asset = config.assets.find((candidate) => candidate.id === id && candidate.kind === "image");
+ if (!asset || (!asset.relativePath && !asset.sourcePath)) return;
+ const referenceId = `ref-${crypto.randomUUID()}`;
+ const reference: ProjectReference = {
+ id: referenceId, kind: "text", name: asset.name, description: "", intendedUse: [], createdAt: new Date().toISOString(),
+ images: [{ id: `${referenceId}-image`, name: asset.name, relativePath: asset.relativePath, sourcePath: asset.sourcePath }],
+ };
+ const job = asScene ? createDraftGenerationJob("", {
+ title: asset.name, sceneType: "first-last-frame", startFrameReferenceId: referenceId,
+ steps: defaultGeneratorTemplate().defaultSteps,
+ }) : null;
+ onChange((current) => ({ ...current, references: [reference, ...current.references],
+ generationJobs: job ? [...current.generationJobs, job] : current.generationJobs }));
+ if (job) onOpenGenerator?.(job.id);
+ else onOpenReferences?.();
+ };
const replaceScene = (next: ImageScene) => {
undo.current = []; redo.current = []; historyChanged((value) => value + 1);
pointer.current = null; setDraftBox(null); setDrawKind(null); setCollapsed(new Set());
@@ -487,7 +508,10 @@ export function ImageEditor({ config, folderPath, onChange: changeConfig, onGene
}) },
...(imageMenu.id ? [
{ label: "Edit", icon: <Edit16 />, action: () => attempt(() => config.assets.find((asset) => asset.id === imageMenu.id)?.imageDraft ? selectImage(imageMenu.id!) : replaceScene(editGeneratedImage(config, imageMenu.id!).imageScene!)) },
+ { label: "Create Scene", icon: <Film16 />, separator: true, disabled: !canUseMenuImage, action: () => attempt(() => createFromImage(imageMenu.id!, true)) },
+ { label: "Create Reference", icon: <References16 />, disabled: !canUseMenuImage, action: () => attempt(() => createFromImage(imageMenu.id!, false)) },
{ label: "Remove", icon: <Delete16 />, danger: true,
+ separator: true,
disabled: workItems.some((item) => isWorkActive(item) && item.imageAssetId === imageMenu.id) || (!imageMenu.inFamily && imageFamily(images, imageMenu.id).length > 1),
action: () => removeImage(imageMenu.id!) },
] : []),