Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

20.2 KB
Raw Blame
import { mkdtemp, readFile, rm, utimes, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterAll, describe, expect, it } from 'vitest';
import sharp from 'sharp';
import {
  detectSolidBorderColor,
  hasMeaningfulAlpha,
  keySolidBackground,
  KEYING_CACHE_VERSION,
  prepareSourceImage,
  subjectFrameLayout,
} from './sourceKeying.js';
import { sha256File } from '../../lib/fileUtils.js';

// Build a raw RGBA buffer from a painter function (x, y) => [r, g, b, a?].
const makeImage = (width, height, paint) => {
  const data = new Uint8Array(width * height * 4);
  for (let y = 0; y < height; y += 1) {
    for (let x = 0; x < width; x += 1) {
      const [r, g, b, a = 255] = paint(x, y);
      const i = (y * width + x) * 4;
      data[i] = r; data[i + 1] = g; data[i + 2] = b; data[i + 3] = a;
    }
  }
  return { data, width, height };
};

const GREEN = [30, 200, 40];
const BROWN = [140, 90, 50];

// A 20×20 "subject on green screen": a 10×10 brown block centered on flat green.
const subjectOnGreen = () => makeImage(20, 20, (x, y) => (
  x >= 5 && x < 15 && y >= 5 && y < 15 ? BROWN : GREEN
));

// A 20×20 image with no dominant border color (photo-like busy background).
const noisyImage = () => makeImage(20, 20, (x, y) => [
  (x * 37 + y * 91) % 256, (x * 13 + y * 7) % 256, (x * 71 + y * 3) % 256,
]);

const alphaAt = (image, x, y) => image.data[(y * image.width + x) * 4 + 3];

describe('detectSolidBorderColor', () => {
  it('detects a flat border color', () => {
    expect(detectSolidBorderColor(subjectOnGreen())).toEqual(GREEN);
  });

  it('returns null for a busy border', () => {
    expect(detectSolidBorderColor(noisyImage())).toBeNull();
  });

  it('tolerates minor border noise within the match tolerance', () => {
    const slightlyNoisy = makeImage(20, 20, (x, y) => (
      x >= 5 && x < 15 && y >= 5 && y < 15
        ? BROWN
        : [GREEN[0] + ((x + y) % 5), GREEN[1] - ((x * y) % 5), GREEN[2]]
    ));
    const color = detectSolidBorderColor(slightlyNoisy);
    expect(color).not.toBeNull();
  });
});

describe('keySolidBackground', () => {
  it('keys the background transparent and leaves the subject opaque', () => {
    const result = keySolidBackground(subjectOnGreen());
    expect(result).not.toBeNull();
    const keyed = { data: result.data, width: 20, height: 20 };
    expect(alphaAt(keyed, 0, 0)).toBe(0); // background corner
    expect(alphaAt(keyed, 10, 10)).toBe(255); // subject center
    expect(alphaAt(keyed, 6, 6)).toBe(255); // subject interior near the edge
  });

  it('floods through a dirty outer row using the shared border-band seeds', () => {
    const dirtyEdge = makeImage(20, 20, (x, y) => (
      y === 0 ? [255, 0, 0] : (x >= 5 && x < 15 && y >= 5 && y < 15 ? BROWN : GREEN)
    ));
    const result = keySolidBackground(dirtyEdge);
    expect(result).not.toBeNull();
    const keyed = { data: result.data, width: 20, height: 20 };
    expect(alphaAt(keyed, 0, 1)).toBe(0);
    expect(alphaAt(keyed, 0, 0)).toBe(255); // non-background edge artifact is preserved
  });

  it('keys enclosed background pockets the flood fill cannot reach', () => {
    // A brown ring with a green hole in the middle — unreachable from the border.
    const ring = makeImage(20, 20, (x, y) => {
      const inRing = x >= 4 && x < 16 && y >= 4 && y < 16;
      const inHole = x >= 8 && x < 12 && y >= 8 && y < 12;
      if (inHole) return GREEN;
      if (inRing) return BROWN;
      return GREEN;
    });
    const result = keySolidBackground(ring);
    expect(result).not.toBeNull();
    const keyed = { data: result.data, width: 20, height: 20 };
    expect(alphaAt(keyed, 10, 10)).toBe(0); // the enclosed pocket
    expect(alphaAt(keyed, 5, 10)).toBe(255); // the ring itself
  });

  it('returns null when the whole image is background (nothing to keep)', () => {
    expect(keySolidBackground(makeImage(20, 20, () => GREEN))).toBeNull();
  });

  it('returns null when there is no solid background to key', () => {
    expect(keySolidBackground(noisyImage())).toBeNull();
  });
});

describe('hasMeaningfulAlpha', () => {
  it('distinguishes an all-opaque alpha channel from a real one', () => {
    expect(hasMeaningfulAlpha(subjectOnGreen().data)).toBe(false);
    const withAlpha = makeImage(4, 4, (x) => [0, 0, 0, x === 0 ? 0 : 255]);
    expect(hasMeaningfulAlpha(withAlpha.data)).toBe(true);
  });
});

describe('prepareSourceImage', () => {
  let dir;
  const tempDir = async () => {
    dir = dir || await mkdtemp(join(tmpdir(), 'portos-keying-'));
    return dir;
  };
  afterAll(async () => { if (dir) await rm(dir, { recursive: true, force: true }); });

  const writePng = async (name, image) => {
    const path = join(await tempDir(), name);
    await sharp(Buffer.from(image.data), {
      raw: { width: image.width, height: image.height, channels: 4 },
    }).png().toFile(path);
    return path;
  };

  it('returns null for a transparent source — the pipeline uses real alpha directly', async () => {
    const withAlpha = makeImage(20, 20, (x, y) => (
      x >= 5 && x < 15 && y >= 5 && y < 15 ? [...BROWN, 255] : [0, 0, 0, 0]
    ));
    const sourcePath = await writePng('transparent.png', withAlpha);
    const result = await prepareSourceImage({ sourcePath, targetPath: join(await tempDir(), 'never.png') });
    expect(result).toBeNull();
  });

  it('keys a solid-background source into the target path and returns it', async () => {
    const sourcePath = await writePng('green.png', subjectOnGreen());
    const targetPath = join(await tempDir(), 'keyed.png');
    const result = await prepareSourceImage({ sourcePath, targetPath });
    expect(result).toEqual({ path: targetPath, keyed: true, framed: false });

    const { data, info } = await sharp(targetPath).ensureAlpha().raw()
      .toBuffer({ resolveWithObject: true });
    const keyed = { data, width: info.width, height: info.height };
    expect(alphaAt(keyed, 0, 0)).toBe(0);
    expect(alphaAt(keyed, 10, 10)).toBe(255);
  });

  it('returns null for a busy-background source — the pipeline’s own matting applies', async () => {
    const noisy = noisyImage();
    const sourcePath = await writePng('noisy.png', noisy);
    const result = await prepareSourceImage({ sourcePath, targetPath: join(await tempDir(), 'never2.png') });
    expect(result).toBeNull();
  });

  it('returns null for a source over the pixel cap without decoding it', async () => {
    // Just over KEY_MAX_PIXELS (16 MP): 4020×4000. Solid color, so the PNG
    // itself is tiny — only the header is read before the cap bails.
    const sourcePath = join(await tempDir(), 'huge.png');
    await sharp({
      create: { width: 4020, height: 4000, channels: 4, background: { r: 30, g: 200, b: 40, alpha: 1 } },
    }).png().toFile(sourcePath);
    const result = await prepareSourceImage({ sourcePath, targetPath: join(await tempDir(), 'never3.png') });
    expect(result).toBeNull();
  });

  it('reuses a keyed target newer than the gallery source', async () => {
    const sourcePath = await writePng('cache-source.png', subjectOnGreen());
    const cachedTarget = await writePng('cache-target.png', makeImage(20, 20, () => [255, 0, 0, 255]));
    const sourceSha256 = await sha256File(sourcePath);
    await writeFile(`${cachedTarget}.meta.json`, JSON.stringify({
      version: KEYING_CACHE_VERSION,
      sourceSha256,
      keyBackground: true,
      subjectScale: 1,
      keyed: true,
      framed: false,
    }));
    const fresh = new Date(Date.now() + 60_000);
    await utimes(cachedTarget, fresh, fresh);

    const result = await prepareSourceImage({ sourcePath, targetPath: cachedTarget });
    expect(result).toEqual({ path: cachedTarget, keyed: true, framed: false });
    const { data } = await sharp(cachedTarget).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
    expect([...data.slice(0, 4)]).toEqual([255, 0, 0, 255]);
  });

  it('rekeys when source bytes change despite an older source mtime', async () => {
    const sourcePath = await writePng('fingerprint-source.png', subjectOnGreen());
    const cachedTarget = await writePng('fingerprint-target.png', makeImage(20, 20, () => [255, 0, 0, 255]));
    await writeFile(`${cachedTarget}.meta.json`, JSON.stringify({
      version: KEYING_CACHE_VERSION,
      sourceSha256: await sha256File(sourcePath),
      keyBackground: true,
      subjectScale: 1,
      keyed: true,
      framed: false,
    }));
    const targetTime = new Date(Date.now() + 60_000);
    await utimes(cachedTarget, targetTime, targetTime);

    await writePng('fingerprint-source.png', makeImage(20, 20, (x, y) => (
      x >= 6 && x < 14 && y >= 6 && y < 14 ? [90, 80, 200] : GREEN
    )));
    const oldSourceTime = new Date(Date.now() - 60_000);
    await utimes(sourcePath, oldSourceTime, oldSourceTime);

    const result = await prepareSourceImage({ sourcePath, targetPath: cachedTarget });
    expect(result).toEqual({ path: cachedTarget, keyed: true, framed: false });
    const { data } = await sharp(cachedTarget).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
    expect(alphaAt({ data, width: 20, height: 20 }, 0, 0)).toBe(0);
    expect(JSON.parse(await readFile(`${cachedTarget}.meta.json`, 'utf8')).sourceSha256)
      .toBe(await sha256File(sourcePath));
  });

  it('recomputes a fresh target when its keying-version metadata is stale', async () => {
    const sourcePath = await writePng('version-source.png', subjectOnGreen());
    const targetPath = await writePng('version-target.png', makeImage(20, 20, () => [255, 0, 0, 255]));
    await writeFile(`${targetPath}.meta.json`, JSON.stringify({ version: 'source-keying-old' }));
    const fresh = new Date(Date.now() + 60_000);
    await utimes(targetPath, fresh, fresh);

    const result = await prepareSourceImage({ sourcePath, targetPath });
    expect(result).toEqual({ path: targetPath, keyed: true, framed: false });
    const { data } = await sharp(targetPath).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
    expect(alphaAt({ data, width: 20, height: 20 }, 0, 0)).toBe(0);
    expect(JSON.parse(await readFile(`${targetPath}.meta.json`, 'utf8'))).toEqual({
      version: KEYING_CACHE_VERSION,
      sourceSha256: await sha256File(sourcePath),
      keyBackground: true,
      subjectScale: 1,
      keyed: true,
      framed: false,
    });
  });
});

describe('subjectFrameLayout', () => {
  // The margin is the whole product outcome: a subject at 65% of a square canvas
  // leaves ~17.5% of the canvas empty beyond each of its extremities, which is the
  // context the decoder needs to resolve a fingertip instead of clipping it.
  it('centers a landscape source on a canvas of its longest side', () => {
    expect(subjectFrameLayout({ width: 100, height: 50, subjectScale: 0.6 })).toEqual({
      canvasSize: 100, innerWidth: 60, innerHeight: 30, left: 20, top: 35,
    });
  });

  it('centers a portrait source with the same margins, transposed', () => {
    expect(subjectFrameLayout({ width: 50, height: 100, subjectScale: 0.6 })).toEqual({
      canvasSize: 100, innerWidth: 30, innerHeight: 60, left: 35, top: 20,
    });
  });

  it('never upsamples — the canvas is the source’s longest side, not a fixed size', () => {
    const { canvasSize, innerWidth } = subjectFrameLayout({ width: 37, height: 12, subjectScale: 1 });
    expect(canvasSize).toBe(37);
    expect(innerWidth).toBe(37);
  });

  it('floors both dimensions at 1px so a thin source can’t produce an invalid resize', () => {
    // 4 x 0.05 rounds to 0, which sharp rejects outright.
    expect(subjectFrameLayout({ width: 400, height: 4, subjectScale: 0.05 }))
      .toMatchObject({ innerWidth: 20, innerHeight: 1 });
  });
});

describe('prepareSourceImage subject framing', () => {
  let dir;
  const tempDir = async () => {
    dir = dir || await mkdtemp(join(tmpdir(), 'portos-framing-'));
    return dir;
  };
  afterAll(async () => { if (dir) await rm(dir, { recursive: true, force: true }); });

  const writePng = async (name, image) => {
    const path = join(await tempDir(), name);
    await sharp(Buffer.from(image.data), {
      raw: { width: image.width, height: image.height, channels: 4 },
    }).png().toFile(path);
    return path;
  };

  const readRgba = async (path) => {
    const { data, info } = await sharp(path).ensureAlpha().raw()
      .toBuffer({ resolveWithObject: true });
    return { data, width: info.width, height: info.height };
  };

  const pixelAt = (image, x, y) => {
    const i = (y * image.width + x) * 4;
    return [image.data[i], image.data[i + 1], image.data[i + 2], image.data[i + 3]];
  };

  // A 40x20 landscape photo-like source: no alpha, and a busy border so keying
  // would decline it even if it were asked for.
  const landscapePhoto = () => makeImage(40, 20, (x, y) => [
    (x * 37 + y * 91) % 256, (x * 13 + y * 7) % 256, (x * 71 + y * 3) % 256,
  ]);

  it('is a pass-through at the default scale with keying off — the original is used', async () => {
    // The strongest form of "byte-identical": nothing is written at all, so the
    // render consumes the untouched gallery file.
    const sourcePath = await writePng('identity.png', landscapePhoto());
    const targetPath = join(await tempDir(), 'identity-out.png');
    expect(await prepareSourceImage({
      sourcePath, targetPath, keyBackground: false, subjectScale: 1,
    })).toBeNull();
    await expect(readFile(targetPath)).rejects.toThrow();
  });

  it.each([0, 1.5, -0.2, NaN, '0.5', null])(
    'treats the out-of-range scale %s as the identity rather than framing on it',
    async (bad) => {
      const sourcePath = await writePng(`bad-${String(bad)}.png`, landscapePhoto());
      expect(await prepareSourceImage({
        sourcePath,
        targetPath: join(await tempDir(), `bad-${String(bad)}-out.png`),
        keyBackground: false,
        subjectScale: bad,
      })).toBeNull();
    },
  );

  it('frames an opaque source on a square canvas without keying it', async () => {
    const sourcePath = await writePng('frame-opaque.png', landscapePhoto());
    const targetPath = join(await tempDir(), 'frame-opaque-out.png');
    const result = await prepareSourceImage({
      sourcePath, targetPath, keyBackground: false, subjectScale: 0.5,
    });
    expect(result).toEqual({ path: targetPath, keyed: false, framed: true });

    const framed = await readRgba(targetPath);
    // Canvas = the source's longest side (40), subject 20x10 centered at (10, 15).
    expect([framed.width, framed.height]).toEqual([40, 40]);
    // An opaque source must stay opaque: a transparent pad would read to TRELLIS.2
    // as "already matted" and silently disable its own background removal.
    expect(pixelAt(framed, 0, 0)[3]).toBe(255);
    expect(pixelAt(framed, 20, 20)[3]).toBe(255);
    // The margin is real: the corners are pad, not subject.
    expect(pixelAt(framed, 1, 1).slice(0, 3)).toEqual([255, 255, 255]);
  });

  it('pads with the source’s own solid border color, so the margin is invisible', async () => {
    const sourcePath = await writePng('frame-green.png', subjectOnGreen());
    const targetPath = join(await tempDir(), 'frame-green-out.png');
    const result = await prepareSourceImage({
      sourcePath, targetPath, keyBackground: false, subjectScale: 0.5,
    });
    expect(result).toEqual({ path: targetPath, keyed: false, framed: true });
    const framed = await readRgba(targetPath);
    expect(pixelAt(framed, 0, 0)).toEqual([...GREEN, 255]);
  });

  it('preserves alpha for an alpha-bearing source and pads transparent', async () => {
    // A transparent source is already matted, so keying declines it — but framing
    // still applies, and the pad MUST stay transparent or the matte is undone.
    const withAlpha = makeImage(20, 20, (x, y) => (
      x >= 5 && x < 15 && y >= 5 && y < 15 ? [...BROWN, 255] : [0, 0, 0, 0]
    ));
    const sourcePath = await writePng('frame-alpha.png', withAlpha);
    const targetPath = join(await tempDir(), 'frame-alpha-out.png');
    const result = await prepareSourceImage({
      sourcePath, targetPath, keyBackground: true, subjectScale: 0.5,
    });
    expect(result).toEqual({ path: targetPath, keyed: false, framed: true });

    const framed = await readRgba(targetPath);
    expect([framed.width, framed.height]).toEqual([20, 20]);
    expect(pixelAt(framed, 0, 0)[3]).toBe(0);
    // Subject center: the 10x10 block resized to 5x5 and centered at (5, 5). Near-
    // rather than fully-opaque because a 2x downscale of a hard alpha edge on a
    // 5px block resamples some of the transparent surround into it — the contrast
    // that matters is against the fully-transparent pad above.
    expect(pixelAt(framed, 10, 10)[3]).toBeGreaterThan(200);
  });

  it('keys first and frames second, so the flood fill still samples the real border', async () => {
    // Order is not interchangeable: framing first would move the border pixels the
    // keyer measures, and its "is this background solid" gate would be reading the
    // pad color instead of the image's.
    const sourcePath = await writePng('key-then-frame.png', subjectOnGreen());
    const targetPath = join(await tempDir(), 'key-then-frame-out.png');
    const result = await prepareSourceImage({
      sourcePath, targetPath, keyBackground: true, subjectScale: 0.5,
    });
    expect(result).toEqual({ path: targetPath, keyed: true, framed: true });

    const framed = await readRgba(targetPath);
    // Keyed → the pad joins the keyed background as transparent, subject stays opaque.
    expect(pixelAt(framed, 0, 0)[3]).toBe(0);
    expect(pixelAt(framed, 10, 10)[3]).toBeGreaterThan(200);
  });

  it('measures an EXIF-rotated source after transposition', async () => {
    // Stored 40x20 (left half red, right half blue) with orientation 6 — a 90° CW
    // display rotation, so the viewer sees 20x40 with red on TOP. Framing on the
    // STORED dimensions would put the subject in a landscape band and leave the
    // sampled points below on empty pad, and the PNG written here carries no
    // orientation tag of its own to fix it downstream.
    const sourcePath = join(await tempDir(), 'exif.jpg');
    await sharp(Buffer.from(makeImage(40, 20, (x) => (x < 20 ? [220, 20, 20] : [20, 20, 220])).data), {
      raw: { width: 40, height: 20, channels: 4 },
    }).withMetadata({ orientation: 6 }).jpeg({ quality: 100 }).toFile(sourcePath);

    const targetPath = join(await tempDir(), 'exif-out.png');
    const result = await prepareSourceImage({
      sourcePath, targetPath, keyBackground: false, subjectScale: 0.5,
    });
    expect(result).toEqual({ path: targetPath, keyed: false, framed: true });

    const framed = await readRgba(targetPath);
    expect([framed.width, framed.height]).toEqual([40, 40]);
    // Subject 10x20 centered at (15, 10): red in its top half, blue in its bottom.
    const [topR, , topB] = pixelAt(framed, 20, 13);
    const [botR, , botB] = pixelAt(framed, 20, 27);
    expect(topR).toBeGreaterThan(topB);
    expect(botB).toBeGreaterThan(botR);
  });

  it('re-frames rather than reusing the cache when the scale changes', async () => {
    // KEYING_CACHE_VERSION alone cannot carry a PER-RUN parameter, so the framing
    // request is part of the cache identity in the metadata. Without this a
    // re-render at a new scale silently serves the previously prepared image.
    const sourcePath = await writePng('cache-scale.png', landscapePhoto());
    const targetPath = join(await tempDir(), 'cache-scale-out.png');

    await prepareSourceImage({ sourcePath, targetPath, keyBackground: false, subjectScale: 0.5 });
    const first = await readFile(targetPath);
    expect(JSON.parse(await readFile(`${targetPath}.meta.json`, 'utf8'))).toMatchObject({
      version: KEYING_CACHE_VERSION, keyBackground: false, subjectScale: 0.5, framed: true,
    });

    await prepareSourceImage({ sourcePath, targetPath, keyBackground: false, subjectScale: 0.8 });
    expect(await readFile(targetPath)).not.toEqual(first);
    expect(JSON.parse(await readFile(`${targetPath}.meta.json`, 'utf8')))
      .toMatchObject({ subjectScale: 0.8 });

    // And the SAME request is still served from cache — the identity check must not
    // have degenerated into "always recompute".
    const fresh = new Date(Date.now() + 60_000);
    await utimes(targetPath, fresh, fresh);
    const cached = await readFile(targetPath);
    await prepareSourceImage({ sourcePath, targetPath, keyBackground: false, subjectScale: 0.8 });
    expect(await readFile(targetPath)).toEqual(cached);
  });
});