Skip to content

AI media harness for planning, generating, and editing locally

TypeScript
14.7 KB
Raw Blame
import { expect, it } from "vitest";
import { alignExtendBounds, compileExtendImage, extendContext, extendLayout, extendRegions, extendSourceDimensions, EXTEND_MAX_EDGE, EXTEND_MAX_PIXELS, initialExtendBounds } from "./extendImage";
import { createProjectConfig } from "./project";

it("partitions only new pixels for every combination of extension directions and crops", () => {
  for (const x of [-3, 0, 2]) for (const y of [-2, 0, 1]) for (const width of [5, 10]) for (const height of [4, 8]) {
    const { preserved, regions } = extendRegions({ width: 7, height: 5 }, { x, y, width, height });
    for (let py = 0; py < height; py++) for (let px = 0; px < width; px++) {
      const inside = px + x >= 0 && px + x < 7 && py + y >= 0 && py + y < 5;
      const contains = (r: typeof preserved) => px >= r.x && py >= r.y && px < r.x + r.width && py < r.y + r.height;
      expect(contains(preserved)).toBe(inside);
      expect(regions.filter(contains)).toHaveLength(inside ? 0 : 1);
    }
  }
});

it("rejects boxes with no original context, fractional or excessive dimensions", () => {
  for (const bounds of [
    { x: 100, y: 0, width: 20, height: 20 }, { x: -20, y: 0, width: 20, height: 20 },
    { x: -1.5, y: 0, width: 20, height: 20 },
    { x: -20, y: 0, width: 8193, height: 20 }, { x: NaN, y: 0, width: 20, height: 20 },
  ]) expect(() => extendRegions({ width: 100, height: 100 }, bounds)).toThrow();
  const large = initialExtendBounds(8192, 400);
  expect(large.width).toBe(EXTEND_MAX_EDGE);
  expect(large.width % 32).toBe(0);
  expect(large.height % 32).toBe(0);
  expect(large.x).toBeLessThan(0);
});

it("generates the entire surround in one pass while preserving the original in all directions", () => {
  const config = createProjectConfig({ name: "Extend", prompt: "", aspectRatio: "1:1", resolution: "416p", targetDurationSeconds: 15 });
  const source = { id: "source", name: "Source", kind: "image" as const, mimeType: "image/png", width: 128, height: 96, createdAt: config.createdAt };
  for (const x of [-64, 0, 32]) for (const y of [-32, 0, 32]) {
    const { layout, edits } = compileExtendImage(config, source, "cache/canvas.png", { bounds: { x, y, width: 192, height: 128 }, prompt: "Forest", steps: 20, seed: 0 });
    expect(edits).toHaveLength(1);
    expect(edits[0]).toMatchObject({ ...extendContext(layout.preserved), invertMask: true, feather: 0, outpaintBlendOverlap: 33, outpaintLangevinSteps: 5 });
    const contains = (r: { x: number; y: number; width: number; height: number }, px: number, py: number) => px >= r.x && py >= r.y && px < r.x + r.width && py < r.y + r.height;
    let allNewPixelsGenerated = true;
    for (let py = 0; py < layout.output.height; py++) for (let px = 0; px < layout.output.width; px++) {
      const generated = edits[0].invertMask && !contains(edits[0], px, py);
      if (!contains(layout.preserved, px, py) && !generated) allNewPixelsGenerated = false;
    }
    expect(allNewPixelsGenerated).toBe(true);
  }
});

it("uses the positioned canvas as context and reserves picture labels only for cited references", () => {
  const config = createProjectConfig({ name: "Extend", prompt: "", aspectRatio: "1:1", resolution: "768p", targetDurationSeconds: 15 });
  const source = { id: "source", name: "Source", kind: "image" as const, mimeType: "image/png", width: 768, height: 768, createdAt: config.createdAt };
  const options = { bounds: { x: 0, y: 0, width: 1152, height: 1152 }, prompt: "", steps: 20, seed: 0 };
  const compile = (prompt: string) => compileExtendImage(config, source, "cache/canvas.png", { ...options, prompt });
  const withoutReferences = compile("");
  expect(withoutReferences.prompt).toMatch(/^integrated_multimodal_description:/);
  expect(withoutReferences.prompt).not.toContain("<Picture");
  expect(withoutReferences.prompt).toContain("Source scene's setting");
  expect(withoutReferences.prompt).not.toMatch(/zoomed out|continues above|continues below/);
  expect(withoutReferences.edits.every((edit) => edit.prompt === withoutReferences.prompt)).toBe(true);
  config.references = [{ id: "mood", name: "Mood", kind: "text", content: "Misty woodland", description: "", intendedUse: [], createdAt: config.createdAt }];
  expect(compile("@[ref:mood]").prompt).not.toContain("<Picture");
  for (let index = 1; index <= 9; index++) config.references.push({
    id: `ref-${index}`, name: `Tree ${index}`, kind: "image", relativePath: `references/${index}.png`, description: "", intendedUse: [], createdAt: config.createdAt,
  });
  const withReferences = compile(config.references.map((reference) => `@[ref:${reference.id}]`).join(" "));
  expect(withReferences.references).toHaveLength(10);
  for (let index = 1; index <= 9; index++) {
    expect(withReferences.prompt).toContain(`<Subject ${index + 1}> is Tree ${index}, providing appearance from <Picture ${index}>`);
  }
  expect(withReferences.prompt).not.toContain("<Picture 10>");
});

it("rejects an extension with too little preserved image to condition the generation", () => {
  const config = createProjectConfig({ name: "Extend", prompt: "", aspectRatio: "1:1", resolution: "768p", targetDurationSeconds: 15 });
  const source = { id: "source", name: "Source", kind: "image" as const, mimeType: "image/png", width: 128, height: 96, createdAt: config.createdAt };
  expect(() => compileExtendImage(config, source, "cache/canvas.png", {
    bounds: { x: 127, y: 0, width: 192, height: 192 }, prompt: "", steps: 20, seed: 0,
  })).toThrow("Keep a larger area");
});

it("uses the selected output budget independently of source size, within the generation grid", () => {
  const config = createProjectConfig({ name: "Extend", prompt: "", aspectRatio: "16:9", resolution: "2048p", targetDurationSeconds: 15 });
  for (const source of [{ width: 400, height: 300 }, { width: 3648, height: 2048 }, { width: 2048, height: 3648 }, { width: 8192, height: 8192 }]) {
    const bounds = initialExtendBounds(source.width, source.height);
    const layout = extendLayout(config, source, bounds);
    for (const size of [bounds, layout.output]) {
      expect(size.width % 32).toBe(0); expect(size.height % 32).toBe(0);
      expect(Math.max(size.width, size.height)).toBeLessThanOrEqual(EXTEND_MAX_EDGE);
      expect(size.width * size.height).toBeLessThanOrEqual(EXTEND_MAX_PIXELS);
    }
    expect(layout.output.width * layout.output.height).toBeGreaterThan(EXTEND_MAX_PIXELS * 0.96);
    expect(layout.source.width).toBeLessThan(layout.output.width);
    expect(layout.source.height).toBeLessThan(layout.output.height);
    expect(Math.abs(layout.source.width / source.width - layout.source.height / source.height)).toBeLessThan(1 / Math.min(source.width, source.height));
    expect(layout.regions).toHaveLength(4);
    const workspace = extendSourceDimensions(source.width, source.height);
    expect(bounds.x + bounds.width).toBeGreaterThan(workspace.width);
    expect(bounds.y + bounds.height).toBeGreaterThan(workspace.height);
  }
});

it("uses the selected resolution budget and the same scaled geometry for preservation and inpainting", () => {
  const config = createProjectConfig({ name: "Extend", prompt: "", aspectRatio: "1:1", resolution: "416p", targetDurationSeconds: 15 });
  const source = { id: "source", name: "Source", kind: "image" as const, mimeType: "image/png", width: 128, height: 96, createdAt: config.createdAt };
  const options = { bounds: { x: -64, y: -32, width: 256, height: 160 }, prompt: "Forest", steps: 20, seed: 0 };
  const compiled = compileExtendImage(config, source, "cache/source.png", options);
  expect(compiled.layout).toMatchObject({ output: { width: 512, height: 320 }, source: { width: 256, height: 192 }, bounds: { x: -128, y: -64, width: 512, height: 320 }, preserved: { x: 128, y: 64, width: 256, height: 192 } });
  expect(compiled.scene.sourceImage).toMatchObject({ width: 512, height: 320 });
  expect(compiled.edits[0]).toMatchObject({ x: 128, y: 64, width: 256, height: 192 });
  const large = { width: 2048, height: 2048 };
  const layout = extendLayout(config, large, initialExtendBounds(large.width, large.height));
  expect(layout.output.width * layout.output.height).toBeLessThanOrEqual(416 * 416);
  expect(() => compileExtendImage(config, source, "cache/source.png", { ...options, bounds: { ...options.bounds, width: 257 } })).toThrow("multiples of 32");
});

it("shrinks the source when the selection expands, and grows output when resolution increases", () => {
  const config = createProjectConfig({ name: "Extend", prompt: "", aspectRatio: "1:1", resolution: "768p", targetDurationSeconds: 15 });
  const source = { width: 768, height: 768 };
  const small = extendLayout(config, source, { x: -192, y: -192, width: 1152, height: 1152 });
  const large = extendLayout(config, source, { x: -384, y: -384, width: 1536, height: 1536 });
  expect(small.output).toEqual({ width: 768, height: 768 });
  expect(large.output).toEqual(small.output);
  expect(small.source).toEqual({ width: 512, height: 512 });
  expect(large.source).toEqual({ width: 384, height: 384 });
  const higher = extendLayout({ ...config, imageSettings: { ...config.settings, resolution: "1088p" } }, source, { x: -384, y: -384, width: 1536, height: 1536 });
  expect(higher.output).toEqual({ width: 1088, height: 1088 });
  expect(higher.source).toEqual({ width: 544, height: 544 });
});

it("passes the full source rectangle to native blending and retains a positioned patch", () => {
  expect(extendContext({ x: 0, y: 0, width: 256, height: 256 }))
    .toEqual({ x: 0, y: 0, width: 256, height: 256 });
  expect(extendContext({ x: 64, y: 32, width: 256, height: 256 }))
    .toEqual({ x: 64, y: 32, width: 256, height: 256 });
  expect(() => extendContext({ x: 16, y: 16, width: 32, height: 32 })).toThrow("larger area");
});

it("limits the blend for small source intersections without erasing their interior", () => {
  const config = createProjectConfig({ name: "Zoom", prompt: "", aspectRatio: "1:1", resolution: "416p", targetDurationSeconds: 15 });
  for (const width of [32, 64, 96, 128, 256]) {
    const source = { id: "source", name: "Source", kind: "image" as const, mimeType: "image/png", width, height: width, createdAt: config.createdAt };
    const { edits: [edit] } = compileExtendImage(config, source, "cache/canvas.png",
      { bounds: { x: -32, y: -32, width: 416, height: 416 }, prompt: "", steps: 30, seed: 0 });
    if (edit.outpaintBlendOverlap === undefined) throw new Error("Expected an outpaint edit");
    expect(edit.outpaintBlendOverlap).toBeGreaterThanOrEqual(1);
    expect(edit.outpaintBlendOverlap).toBeLessThanOrEqual(33);
    expect(edit.outpaintBlendOverlap % 2).toBe(1);
    expect(edit.width - 2 * (edit.outpaintBlendOverlap - 1)).toBeGreaterThanOrEqual(32);
    expect(edit.height - 2 * (edit.outpaintBlendOverlap - 1)).toBeGreaterThanOrEqual(32);
  }
});

it("aligns resize dimensions while preserving the opposite corner and caps every sizing path", () => {
  expect(alignExtendBounds({ x: -10, y: -7, width: 610, height: 455 }, "nw")).toEqual({ x: -8, y: 0, width: 608, height: 448 });
  for (const mode of ["draw", "nw", "e", "s"]) {
    const bounds = alignExtendBounds({ x: 0, y: 0, width: 9999, height: 9999 }, mode);
    expect(bounds.width % 32).toBe(0); expect(bounds.height % 32).toBe(0);
    expect(bounds.width * bounds.height).toBeLessThanOrEqual(EXTEND_MAX_PIXELS);
    expect(Math.max(bounds.width,bounds.height)).toBeLessThanOrEqual(EXTEND_MAX_EDGE);
  }
});


it("regenerates contained crops and full-image boxes at the target budget", () => {
  const config = createProjectConfig({ name: "Crop", prompt: "", aspectRatio: "1:1", resolution: "768p", targetDurationSeconds: 15 });
  const source = { id: "source", name: "Source", kind: "image" as const, mimeType: "image/png", width: 256, height: 256, createdAt: config.createdAt };
  for (const bounds of [{ x: 32, y: 64, width: 128, height: 96 }, { x: 0, y: 0, width: 256, height: 256 }]) {
    const compiled = compileExtendImage(config, source, "cache/crop.png", { bounds, prompt: "Fine fur", steps: 20, seed: 0 });
    expect(compiled.layout.mode).toBe("regenerate");
    expect(compiled.layout.regions).toEqual([]);
    expect(compiled.edits).toEqual([{ x: 0, y: 0, ...compiled.layout.output, strength: 0.65, invertMask: false, feather: 0, prompt: compiled.prompt }]);
    expect(compiled.layout.output.width * compiled.layout.output.height).toBeLessThanOrEqual(768 * 768);
    expect(compiled.layout.output.width).toBeGreaterThan(bounds.width);
    expect(compiled.prompt).toContain("Refine the selected crop in <Picture 1>");
    expect(compiled.prompt).toContain("Fine fur");
    expect(compiled.scene.sourceImage).toMatchObject(compiled.layout.output);
  }
  const small = extendLayout(config, { width: 8192, height: 8192 }, { x: 100, y: 100, width: 32, height: 32 });
  expect(small).toMatchObject({ mode: "regenerate", source: { width: 768, height: 768 }, bounds: { x: 0, y: 0, width: 768, height: 768 } });
  const crossing = extendLayout(config, source, { x: -1, y: 64, width: 128, height: 96 });
  expect(crossing.mode).toBe("outpaint");
});


it("rejects an extension that disappears when mapped to a lower resolution", () => {
  const config = createProjectConfig({ name: "Extend", prompt: "", aspectRatio: "1:1", resolution: "416p", targetDurationSeconds: 15 });
  expect(() => extendLayout(config, { width: 1024, height: 1024 }, { x: -1, y: 0, width: 1024, height: 1024 })).toThrow("Expand the box further");
});

it("extends Fibo with a border mask and refines crops without H3 conditioning", () => {
  const config = createProjectConfig({ name: "Zoom", prompt: "", aspectRatio: "1:1", resolution: "416p", targetDurationSeconds: 15 });
  const source = { id: "source", name: "Source", kind: "image" as const, mimeType: "image/png", width: 128, height: 96, createdAt: config.createdAt };
  for (const bounds of [{ x: -32, y: -32, width: 192, height: 160 }, { x: 0, y: 0, width: 192, height: 96 }, { x: 32, y: 0, width: 64, height: 64 }]) {
    const result = compileExtendImage(config, source, "cache/canvas.png", { bounds, prompt: "A forest", steps: 4, seed: 1 }, "fibo2");
    const edit = result.edits[0];
    expect(result.references).toEqual([]);
    expect(result.prompt).toContain("A forest");
    expect(result.prompt).not.toMatch(/Picture|Source scene|Shot|overall_soundscape/);
    expect(edit.invertMask).toBe(false);
    expect(edit.outpaintLangevinSteps).toBeUndefined();
    expect(edit.outpaintBlendOverlap).toBeUndefined();
    if (result.layout.mode === "regenerate") {
      expect(edit.strength).toBe(0.65);
      expect(edit.maskContours).toBeUndefined();
    } else {
      const { width, height } = result.layout.output;
      const { x, y, width: w, height: h } = result.layout.preserved;
      expect(edit.maskContours).toEqual([
        [{ x: 0, y: 0 }, { x: width, y: 0 }, { x: width, y: height }, { x: 0, y: height }],
        [{ x, y }, { x: x + w, y }, { x: x + w, y: y + h }, { x, y: y + h }],
      ]);
    }
  }
});