AI media harness for planning, generating, and editing locally
TypeScript
import { FIBO_MODEL_TYPE, MINIMAX_H3_MODEL_TYPE } from "./settings";
import type { ImageEditStep } from "./imageEditing";
import { imageSettings, imageGenerationDimensions, imageGenerationSizeError, H3_IMAGE_MAX_PIXELS } from "./imageSettings";
import { createImageEditScene } from "./imageScene";
import { compileImagePrompt } from "./imagePrompt";
import { outputDimensions } from "./export";
import type { ProjectAsset, ProjectConfig } from "./project";
export const EXTEND_GRID = 32;
export const ZOOM_BLEND_OVERLAP = 33;
export const ZOOM_LANGEVIN_STEPS = 5;
// Largest current generation preset (2048p landscape), also usable in portrait.
const MAX_GENERATION_SIZE = outputDimensions("2048p", "16:9");
export const EXTEND_MAX_EDGE = Math.max(MAX_GENERATION_SIZE.width, MAX_GENERATION_SIZE.height);
export const EXTEND_MAX_PIXELS = MAX_GENERATION_SIZE.width * MAX_GENERATION_SIZE.height;
/** Selection bounds in the Zoom workspace; negative coordinates extend left/up. */
export interface ExtendBounds { x: number; y: number; width: number; height: number }
export interface ExtendOptions { bounds: ExtendBounds; prompt: string; steps: number; seed: number }
export function extendSourceDimensions(width: number, height: number) {
// Reserve room to extend even when the source is already at the model limit.
const scale = Math.min(1, EXTEND_MAX_EDGE / (width * 1.5), EXTEND_MAX_EDGE / (height * 1.5), Math.sqrt(EXTEND_MAX_PIXELS / (width * height)) / 1.5);
return { width: Math.max(1, Math.floor(width * scale)), height: Math.max(1, Math.floor(height * scale)) };
}
export function alignExtendBounds(bounds: ExtendBounds, mode = "se"): ExtendBounds {
const dimension = (size: number) => Math.max(EXTEND_GRID, Math.min(EXTEND_MAX_EDGE, Math.round(size / EXTEND_GRID) * EXTEND_GRID));
let width = dimension(bounds.width), height = dimension(bounds.height);
if (width * height > EXTEND_MAX_PIXELS) {
if (mode === "e" || mode === "w") width = Math.floor(EXTEND_MAX_PIXELS / height / EXTEND_GRID) * EXTEND_GRID;
else if (mode === "n" || mode === "s") height = Math.floor(EXTEND_MAX_PIXELS / width / EXTEND_GRID) * EXTEND_GRID;
else {
const scale = Math.sqrt(EXTEND_MAX_PIXELS / (width * height));
width = Math.floor(width * scale / EXTEND_GRID) * EXTEND_GRID;
height = Math.floor(height * scale / EXTEND_GRID) * EXTEND_GRID;
}
}
return { x: Math.round(bounds.x + (mode.includes("w") && mode !== "draw" ? bounds.width - width : 0)),
y: Math.round(bounds.y + (mode.includes("n") ? bounds.height - height : 0)), width, height };
}
export function initialExtendBounds(width: number, height: number): ExtendBounds {
const source = extendSourceDimensions(width, height);
const box = alignExtendBounds({ x: 0, y: 0, width: source.width * 1.5, height: source.height * 1.5 });
return { ...box, x: Math.round((source.width - box.width) / 2), y: Math.round((source.height - box.height) / 2) };
}
export function extendOutputDimensions(config: ProjectConfig, bounds: ExtendBounds, modelType?: string) {
const selected = imageGenerationDimensions(imageSettings(config).resolution, imageSettings(config).aspectRatio, modelType);
const error = imageGenerationSizeError(selected, modelType);
if (error) throw new Error(error);
const pixelBudget = Math.min(selected.width * selected.height, modelType === "h3_image" ? H3_IMAGE_MAX_PIXELS : EXTEND_MAX_PIXELS);
const maxEdge = modelType === "h3_image" ? 8192 : EXTEND_MAX_EDGE;
const scale = Math.min(Math.sqrt(pixelBudget / (bounds.width * bounds.height)), maxEdge / bounds.width, maxEdge / bounds.height);
const width = Math.floor(bounds.width * scale / EXTEND_GRID) * EXTEND_GRID;
const height = Math.floor(bounds.height * scale / EXTEND_GRID) * EXTEND_GRID;
if (width < EXTEND_GRID || height < EXTEND_GRID) throw new Error("The Zoom box is too narrow for a 32-pixel generation grid at this resolution.");
return { width, height };
}
export function extendLayout(config: ProjectConfig, source: { width: number; height: number }, bounds: ExtendBounds, modelType?: string) {
const workspace = extendSourceDimensions(source.width, source.height);
const selection = extendRegions(workspace, bounds);
if (bounds.width % EXTEND_GRID || bounds.height % EXTEND_GRID || bounds.width > EXTEND_MAX_EDGE || bounds.height > EXTEND_MAX_EDGE
|| bounds.width * bounds.height > EXTEND_MAX_PIXELS) throw new Error("Use box dimensions in multiples of 32 within the maximum generation resolution.");
const output = extendOutputDimensions(config, bounds, modelType);
if (!selection.regions.length) {
const frame = { x: 0, y: 0, ...output };
return { mode: "regenerate" as const, output, source: output, bounds: frame, preserved: frame, regions: [] };
}
const scale = Math.min(output.width / bounds.width, output.height / bounds.height);
// A uniform scale preserves the source aspect ratio. Grid rounding adds a
// little generated space instead of stretching the original to fit.
const scaledSource = { width: Math.max(1, Math.round(workspace.width * scale)), height: Math.max(1, Math.round(workspace.height * scale)) };
const mappedBounds = { x: Math.round(bounds.x * scale - (output.width - bounds.width * scale) / 2),
y: Math.round(bounds.y * scale - (output.height - bounds.height * scale) / 2), ...output };
const mapped = extendRegions(scaledSource, mappedBounds);
if (!mapped.regions.length) throw new Error("Expand the box further or increase the resolution to generate the extension.");
return { mode: "outpaint" as const, output, source: scaledSource, bounds: mappedBounds, ...mapped };
}
export function extendRegions(source: { width: number; height: number }, bounds: ExtendBounds) {
const { x, y, width, height } = bounds;
if (![source.width, source.height, x, y, width, height].every(Number.isSafeInteger)
|| source.width < 1 || source.height < 1 || source.width > 8192 || source.height > 8192
|| width < 1 || height < 1 || width > 8192 || height > 8192 || Math.abs(x) > 8192 || Math.abs(y) > 8192) {
throw new Error("Use whole-pixel bounds and dimensions from 1 to 8192 pixels.");
}
const left = Math.max(0, -x), top = Math.max(0, -y);
const right = Math.min(width, source.width - x), bottom = Math.min(height, source.height - y);
if (right <= left || bottom <= top) throw new Error("The box must overlap the original image.");
const regions = [
{ x: 0, y: 0, width, height: top },
{ x: 0, y: bottom, width, height: height - bottom },
{ x: 0, y: top, width: left, height: bottom - top },
{ x: right, y: top, width: width - right, height: bottom - top },
].filter((region) => region.width > 0 && region.height > 0);
return { preserved: { x: left, y: top, width: right - left, height: bottom - top }, regions };
}
/** SlopFab blends the full source rectangle and refines its surroundings.
* Keep at least one complete patch for positioned source conditioning. */
export function extendContext(context: ExtendBounds) {
if (Math.ceil(context.x / EXTEND_GRID) >= Math.floor((context.x + context.width) / EXTEND_GRID)
|| Math.ceil(context.y / EXTEND_GRID) >= Math.floor((context.y + context.height) / EXTEND_GRID)) {
throw new Error("Keep a larger area of the original image inside the Zoom box, or increase the generation resolution.");
}
return context;
}
function extendBlendOverlap(context: ExtendBounds, output: { width: number; height: number }) {
const room = (start: number, size: number, extent: number) => {
const edges = Number(start > 0) + Number(start + size < extent);
return edges ? (size - EXTEND_GRID) / edges : Infinity;
};
// Blend across two latent cells at exposed edges, but retain an unblended
// interior when only a small intersection of the source is on the canvas.
const band = Math.max(0, Math.min(ZOOM_BLEND_OVERLAP - 1,
room(context.x, context.width, output.width), room(context.y, context.height, output.height)));
return Math.floor(band / 2) * 2 + 1;
}
export function compileExtendImage(config: ProjectConfig, source: ProjectAsset, sourceRelativePath: string, options: ExtendOptions, modelType = MINIMAX_H3_MODEL_TYPE) {
if (!source.width || !source.height) throw new Error("Wait for the source image dimensions to load.");
const layout = extendLayout(config, { width: source.width, height: source.height }, options.bounds, modelType);
const { preserved, output } = layout;
const regenerate = layout.mode === "regenerate";
const fibo = modelType === FIBO_MODEL_TYPE;
const h3Image = modelType === "h3_image";
const context = regenerate || fibo ? preserved : extendContext(preserved);
if (!Number.isInteger(options.steps) || options.steps < 2 || options.steps > 1000) throw new Error("Steps must be a whole number from 2 to 1000.");
if (!Number.isSafeInteger(options.seed) || options.seed < -1) throw new Error("Seed must be -1 for random or a non-negative safe integer.");
const scene = createImageEditScene({ relativePath: sourceRelativePath, name: source.name, ...output });
scene.steps = options.steps; scene.seed = options.seed;
scene.nodes[0].description = (fibo
? regenerate ? "Refine the selected crop, preserving its subjects, composition, lighting and style. " : "Extend the image into the blank borders, continuing its scene, perspective, lighting and textures. Preserve the existing subjects and their positions. "
: regenerate
? "Refine the selected crop in <Picture 1> at the target resolution. Recover natural fine detail and clean textures while preserving the subjects, composition, perspective, lighting and visual style. "
: h3Image ? "Extend <Picture 1> into the blank borders, continuing its scene, perspective, lighting and textures. Preserve the existing subjects and their positions. "
: "One continuous scene matching Source scene's setting, perspective, subject scale, lighting and textures. The existing subjects remain at their canvas positions. ") + options.prompt.trim();
const compiled = compileImagePrompt({ ...config, imageSettings: { ...imageSettings(config), defaultLook: null }, imageScene: scene }, { modelType, ...(!regenerate && !fibo ? { sourceTreatment: "outpaint" as const } : {}) });
// Outpainting locks the interior and generates all borders together. Crops
// refine the whole frame with partial denoising to retain the composition.
const edits: ImageEditStep[] = [{ ...context, prompt: compiled.prompt, feather: 0, invertMask: !regenerate && !fibo,
...(regenerate ? { strength: h3Image ? 1 : 0.65 } : fibo ? {
// Fibo uses an explicit mask: the outer canvas with the source cut out.
// This keeps source pixels intact without H3's inverted-mask conditioning.
x: 0, y: 0, ...output,
maskContours: [{ x: 0, y: 0, ...output }, preserved].map(({ x, y, width, height }) => [
{ x, y }, { x: x + width, y }, { x: x + width, y: y + height }, { x, y: y + height },
]),
} : { outpaintBlendOverlap: h3Image ? 9 : extendBlendOverlap(context, output), outpaintLangevinSteps: h3Image ? 0 : ZOOM_LANGEVIN_STEPS }) }];
return { ...compiled, scene, edits, layout };
}