AI media harness for planning, generating, and editing locally
TypeScript
import { expect, it } from "vitest";
import { parseProjectConfig } from "./project";
import { addImageNode, createImageEditScene } from "./imageScene";
import { compileImageEdits, editGeneratedImage, imageEditSize, imageEditCanvasSize, imageEditRegionSize, limitImageEditScene } from "./imageEditing";
import fixture from "../../fixtures/project-v1-image.json";
it.each([
[1001, 809, 1024, 832], [809, 1001, 832, 1024], [640, 480, 640, 480], [17, 31, 32, 32],
])("aligns a %i × %i source to %i × %i", (width, height, expectedWidth, expectedHeight) => {
expect(imageEditSize({ width, height })).toEqual({ width: expectedWidth, height: expectedHeight });
});
it("keeps local edits in original pixels when the target is smaller and validates saved dimensions", () => {
const config = parseProjectConfig(fixture);
config.imageScene = addImageNode(createImageEditScene({ relativePath: "media/source.png", name: "Source", width: 1001, height: 809 }), "image-root", "object", { x: 750, y: 750, width: 250, height: 250 });
config.imageScene.nodes[1].description = "Add a balloon";
config.imageScene.outputSize = { width: 672, height: 544 };
expect(compileImageEdits(config).edits[0]).toMatchObject({ x: 750, y: 606, width: 251, height: 203 });
expect(imageEditCanvasSize(config.imageScene)).toEqual({ width: 1001, height: 809 });
expect(parseProjectConfig(JSON.parse(JSON.stringify(config))).imageScene).toEqual(config.imageScene);
config.imageScene.outputSize.width = 673;
expect(() => parseProjectConfig(config)).toThrow();
});
it.each([undefined, { width: 4352, height: 6560 }])("always crops local edits on a 4352 by 6529 source with target %j", (outputSize) => {
const config = parseProjectConfig(fixture);
config.imageScene = addImageNode(createImageEditScene({ relativePath: "source.png", name: "Large", width: 4352, height: 6529 }), "image-root", "object", { x: 900, y: 900, width: 100, height: 100 });
config.imageScene.outputSize = outputSize;
config.imageScene.nodes[1].description = "Change the flower";
expect(imageEditCanvasSize(config.imageScene)).toEqual({ width: 4352, height: 6529 });
expect(imageEditRegionSize(config.imageScene)).toEqual({ width: 4352, height: 6560 });
expect(compileImageEdits(config).edits[0]).toMatchObject({ x: 3916, y: 5876, width: 436, height: 653 });
});
it("uses hierarchy order, skips empty groups, and converts boxes to original pixel dimensions", () => {
const config = parseProjectConfig(fixture);
let scene = createImageEditScene({ relativePath: "media/imported/source.png", name: "Source", width: 101, height: 77 });
scene = addImageNode(scene, "image-root", "group");
const group = scene.nodes[1].id;
scene = addImageNode(scene, "image-root", "object");
scene.nodes[2].description = "A blue boat";
scene = addImageNode(scene, group, "object", { x: 900, y: 900, width: 100, height: 100 });
scene.nodes[3].description = "A red balloon";
scene.nodes[3].colors = ["#FF0000"];
config.imageScene = scene;
config.settings.defaultLook = "watercolor";
const { edits } = compileImageEdits(config);
expect(edits).toHaveLength(2);
expect(edits[0]).toMatchObject({ x: 90, y: 69, width: 11, height: 8 });
expect(edits[0].prompt).toContain("A red balloon");
expect(edits[0].prompt).not.toMatch(/Use these colors|#FF0000/);
expect(edits[1]).toMatchObject({ x: 25, y: 19, width: 51, height: 39 });
expect(edits[1].prompt).toContain("A blue boat");
expect(edits[0].prompt).not.toContain("watercolor");
expect(edits[0].prompt).toContain("[Shot 1]");
expect(edits[0].prompt.match(/^\w+:/gm)).toEqual(["subject_definitions:", "summary:", "retention_analysis:", "detailed_description:", "overall_soundscape:", "non_diegetic_music:"]);
expect(edits[0].prompt).toContain("<Picture 1> is the original source image");
expect(edits[0].prompt).toContain("[keyframe completion]");
expect(edits[0].prompt).toContain("<Picture 1> ([Shot 1] edited keyframe): partially_preserved");
expect(edits[0].prompt).toContain("[Shot 1] The edited keyframe corresponds to <Picture 1>");
expect(edits[0].prompt).not.toMatch(/<Subject|video editing|A still photograph/);
for (const edit of edits) expect(edit.prompt).not.toMatch(/\b(box|mask\w*|rectangle|region|boundary|border|pixels?)\b/i);
scene.nodes[3].box = null;
expect(() => compileImageEdits(config)).toThrow("placement box");
});
it("reserves Picture 1 for the source and counts it toward H3's nine-picture limit", () => {
const config = parseProjectConfig(fixture);
config.imageScene = addImageNode(createImageEditScene({ relativePath: "media/imported/source.png", name: "Source", width: 101, height: 77 }), "image-root", "object");
config.imageScene.nodes[1].description = "A red balloon";
config.references = Array.from({ length: 9 }, (_, index) => ({ id: `ref-${index}`, kind: "image" as const, name: `Reference ${index}`, description: "Red satin", relativePath: `references/${index}.png`, intendedUse: [], createdAt: config.createdAt }));
const mention = (count: number) => `A red balloon ${config.references.slice(0, count).map((reference) => `@[ref:${reference.id}]`).join(" ")}`;
config.imageScene.nodes[1].description = mention(8);
const prompt = compileImageEdits(config).edits[0].prompt;
expect(prompt).toContain("<Subject 1> is Reference 0, providing appearance from <Picture 2>");
expect(prompt).toContain("<Subject 8> is Reference 7, providing appearance from <Picture 9>");
expect(prompt).toContain("[keyframe completion + reference generation]");
expect(prompt).not.toMatch(/\b(box|mask\w*|rectangle|region|boundary|border|pixels?)\b/i);
config.imageScene.nodes[1].description = mention(9);
expect(() => compileImageEdits(config)).toThrow("at most nine reference images");
});
it("starts with a clean image root and requires described edits before submission", () => {
const config = parseProjectConfig(fixture);
config.assets.push({ id: "source", name: "Source", relativePath: "media/generated/source.jpg", mimeType: "image/jpeg", width: 101, height: 77, kind: "image", createdAt: config.createdAt });
const edited = editGeneratedImage(config, "source");
expect(edited.imageScene).toMatchObject({ rootType: "image", sourceImage: { width: 101, height: 77 }, outputAssetId: null });
expect(edited.imageScene!.nodes).toHaveLength(1);
expect(edited.imageScene!.nodes[0].description).toBe("");
expect(() => compileImageEdits(edited)).toThrow("Enter a whole-image edit prompt");
edited.imageScene = addImageNode(edited.imageScene!, "image-root", "object");
edited.imageScene.nodes[1].colors = ["#FF0000"];
expect(() => compileImageEdits(edited)).toThrow("Describe the edit");
expect(config.imageScene!.nodes.length).toBeGreaterThan(1);
expect(parseProjectConfig(JSON.parse(JSON.stringify(edited))).imageScene).toEqual(edited.imageScene);
});
it("applies whole-image actions first and compiles only the references each edit cites", () => {
const config = parseProjectConfig(fixture);
config.imageScene = createImageEditScene({ relativePath: "media/source.png", name: "Source", width: 101, height: 77 });
config.references = ["Lighting", "Balloon", "Unused"].map((name) => ({ id: name, name, kind: "image", description: name, relativePath: `references/${name}.png`, intendedUse: [], createdAt: config.createdAt }));
const root = config.imageScene.nodes[0];
root.description = "Warm up the entire image like @[ref:Lighting]";
// Stored ids no longer choose anything: only citations do.
root.referenceIds = ["Unused"];
expect(compileImageEdits(config).edits).toHaveLength(1);
config.imageScene = addImageNode(config.imageScene, root.id, "object");
Object.assign(config.imageScene.nodes[1], { description: "Add a @[ref:Balloon]" });
let edits = compileImageEdits(config).edits;
expect(edits[0]).toMatchObject({ x: 0, y: 0, width: 101, height: 77, feather: 0 });
expect(edits[1].feather).toBeUndefined();
expect(edits[0].prompt).toContain("Warm up the entire image like <Subject 1>");
expect(edits[0].prompt).toContain("<Subject 1> is Lighting, providing appearance from <Picture 2>");
expect(edits[0].prompt).not.toContain("Balloon");
expect(edits[1].prompt).toContain("<Subject 1> is Balloon, providing appearance from <Picture 2>");
expect(edits[1].prompt).not.toContain("Lighting");
expect(edits.map((edit) => edit.references.map((reference) => reference.id))).toEqual([["Lighting"], ["Balloon"]]);
config.imageScene.referenceIds = ["Unused"];
config.imageScene.nodes[1].description = "Add a balloon";
edits = compileImageEdits(config).edits;
expect(edits[1].references).toEqual([]);
config.imageScene.nodes[1].description = "Add a @[ref:missing]";
expect(() => compileImageEdits(config)).toThrow("The prompt cites a reference that no longer exists.");
});
it("caps a local box at the target while keeping feathering against the larger original", () => {
const config = parseProjectConfig(fixture);
config.imageScene = addImageNode(createImageEditScene({ relativePath: "media/source.png", name: "Source", width: 1001, height: 809 }), "image-root", "object", { x: 0, y: 0, width: 1000, height: 1000 });
config.imageScene.outputSize = { width: 672, height: 544 };
config.imageScene.nodes[1].description = "Warm up the lighting";
expect(compileImageEdits(config).edits[0]).toMatchObject({ x: 0, y: 0, width: 672, height: 544 });
expect(compileImageEdits(config).edits[0].feather).toBeUndefined();
// A local edit touching the canvas edge still blends with the rest of the image.
config.imageScene.nodes[1].box!.width = 500;
expect(compileImageEdits(config).edits[0].feather).toBeUndefined();
});
it("bounds lasso outlines and multiple local edits without moving their source positions", () => {
const config = parseProjectConfig(fixture);
let scene = createImageEditScene({ relativePath: "source.png", name: "Large", width: 4096, height: 3072 });
scene.outputSize = { width: 1024, height: 768 };
scene = addImageNode(scene, "image-root", "object", { x: 500, y: 500, width: 500, height: 500 });
Object.assign(scene.nodes[1], { description: "A flower", lasso: [{ x: 0, y: 0 }, { x: 1000, y: 0 }, { x: 0, y: 1000 }] });
scene = addImageNode(scene, "image-root", "object", { x: 100, y: 100, width: 50, height: 50 });
scene.nodes[2].description = "A bee";
config.imageScene = limitImageEditScene(scene);
expect(config.imageScene.nodes[1].box).toEqual({ x: 500, y: 500, width: 250, height: 250 });
const edits = compileImageEdits(config).edits;
expect(edits[0]).toMatchObject({ x: 2048, y: 1536, width: 1024, height: 768,
maskPolygon: [{ x: 2048, y: 1536 }, { x: 3072, y: 1536 }, { x: 2048, y: 2304 }] });
expect(edits[1]).toMatchObject({ x: 409, y: 307, width: 206, height: 154 });
});
it("uses plain Fibo edit instructions, preserves structured JSON and rejects extra pictures", () => {
const config = parseProjectConfig(fixture);
config.imageScene = createImageEditScene({ name: "Source", relativePath: "media/source.png", width: 512, height: 512 });
const compile = () => compileImageEdits(config, { modelType: "fibo2" });
config.imageScene.nodes[0].description = "Make the sky blue";
expect(compile().edits[0]).toMatchObject({ prompt: "Make the sky blue.", references: [], feather: 0 });
const json = '{"edit_instruction":"Make the sky blue","structured_caption":{"sky":"Blue"}}';
config.imageScene.nodes[0].description = json;
expect(compile().edits[0].prompt).toBe(json);
config.references = [{ id: "extra", kind: "image", name: "Extra", relativePath: "references/extra.png", description: "", intendedUse: [], createdAt: config.createdAt }];
config.imageScene.nodes[0].description = "Use @[ref:extra]";
expect(compile).toThrow(/source image only/);
});