Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

20.9 KB
Raw Blame
/**
 * Sprite workflows against a backup cut (#9982). A snapshot copies files and then
 * dumps rows, so the sprite row (status, frozen chroma key) must never be dumped
 * next to files from before the workflow that wrote it. Every workflow that pairs
 * a manifest, walk set or imported tree with its sprite row holds one admission
 * lease from its first byte through the row. The suites hold the row write at
 * its commit seam over real files: a cut requested mid-workflow must wait for
 * it, and a workflow requested mid-cut must not change a byte until the cut ends.
 */
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { mkdtempSync, rmSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { createHash } from 'crypto';
import { mkdir, readFile, stat, writeFile } from 'fs/promises';
import { capSharpThreads, lockAllAnchors, placeCandidate as placeCandidateFixture } from './spriteTestFixtures.js';

const restoreSharpThreads = capSharpThreads();
const TEST_ROOT = mkdtempSync(join(tmpdir(), 'sprite-backup-admission-'));
afterAll(() => {
  restoreSharpThreads();
  rmSync(TEST_ROOT, { recursive: true, force: true });
});

// The file record backend writes every row through atomicWrite, so holding that
// write holds the row commit.
let beforeRowWrite = async () => {};
let afterRunWrite = async () => {};
vi.mock('../../lib/fileUtils.js', async (importOriginal) => {
  const actual = await importOriginal();
  Object.assign(actual.PATHS, {
    data: TEST_ROOT,
    sprites: join(TEST_ROOT, 'sprites'),
    images: join(TEST_ROOT, 'images'),
    videos: join(TEST_ROOT, 'videos'),
  });
  return {
    ...actual,
    atomicWrite: async (path, ...args) => {
      if (path.endsWith('sprite-records.json')) await beforeRowWrite();
      const result = await actual.atomicWrite(path, ...args);
      if (path.endsWith('animation-run.json')) await afterRunWrite(path, args[0]);
      return result;
    },
  };
});

const executeTuiRun = vi.fn(() => new Promise(() => {}));
vi.mock('../tuiPromptRunner.js', () => ({ executeTuiRun: (...args) => executeTuiRun(...args) }));
vi.mock('../../lib/imageCleanDefaults.js', () => ({ resolveImageCleaners: () => ({ cleanC2PA: false, denoise: false }) }));
vi.mock('../settings.js', () => ({
  getSettings: async () => ({ imageGen: { mode: 'grok', grok: { enabled: true, grokPath: '/usr/local/bin/grok' } } }),
}));
vi.mock('../../lib/ffmpeg.js', async (importOriginal) => ({
  ...await importOriginal(),
  probeVideoDuration: async () => null,
}));
const runWalkPostprocess = vi.fn(async ({ runRel, runAbs, recordId, direction, frameCount = 8, fps = 12 }) => {
  await mkdir(join(runAbs, 'generated'), { recursive: true });
  await writeFile(join(runAbs, 'generated', 'strip.png'), 'packaged-strip-bytes');
  await writeFile(join(runAbs, 'generated', 'manifest.json'), JSON.stringify({
    schemaVersion: 1,
    kind: 'deterministically-packaged-grok-walk-video',
    characterId: recordId,
    direction,
    frameCount,
    frameRate: fps,
    stripPath: `${runRel}/generated/strip.png`,
    stripSha256: createHash('sha256').update(Buffer.from('packaged-strip-bytes')).digest('hex'),
  }));
  return {
    manifest: { frameCount, frameRate: fps },
    manifestPath: `${runRel}/generated/manifest.json`,
    stripPreview: { stripPath: `${runRel}/generated/strip.png`, frameCount, fps, cellWidth: 384, cellHeight: 384, row: 0, startColumn: 0 },
  };
});
vi.mock('./walkPostprocess.js', async (importOriginal) => ({
  ...await importOriginal(),
  prepareWalkAnchorChromaInput: async (_anchorAbs, destAbs) => {
    await mkdir(join(destAbs, '..'), { recursive: true });
    await writeFile(destAbs, 'stub-chroma-anchor');
    return { preparation: 'composited-over-solid-chroma-matte', canvas: null };
  },
  runWalkPostprocess: (...args) => runWalkPostprocess(...args),
}));
vi.mock('../mediaJobQueue/index.js', async (importOriginal) => ({
  ...await importOriginal(),
  enqueueJob: () => ({ jobId: 'job-1', position: 0, status: 'queued' }),
}));

const { acquireBackupSnapshotCut } = await import('../../lib/backupSnapshotBoundary.js');
const records = await import('./records.js');
const { buildSpriteRecord } = await import('./recordsLogic.js');
const {
  lockReference, unlockReferenceAnchor, unlockReferenceMain, unlockReferenceTurnaround,
} = await import('./reference.js');
const {
  approveWalkDirection, unlockWalkSet, reopenWalkDirection, invalidateWalkDirectionForAnchorRevision,
  unlockTurnaroundReference, startWalkGeneration,
} = await import('./walk.js');
const { startTrackGeneration } = await import('./animationTrackWorkflow.js');
const { importFromSource } = await import('./importer.js');
const { SPRITE_DIRECTIONS, ANCHOR_DIRECTIONS } = await import('./prompts.js');

const deferred = () => {
  let resolve;
  const promise = new Promise((done) => { resolve = done; });
  return { promise, resolve };
};
const settle = () => new Promise((resolve) => setTimeout(resolve, 40));
const exists = (path) => stat(path).then(() => true, (error) => {
  if (error.code === 'ENOENT') return false;
  throw error;
});
const spritePath = (id, rel) => join(TEST_ROOT, 'sprites', id, rel);
const readJson = async (path) => JSON.parse(await readFile(path, 'utf8'));

/** Hold the next sprite row write; resolves `reached` once the workflow is parked on it. */
function holdRowWrite() {
  const reached = deferred();
  const commit = deferred();
  beforeRowWrite = async () => {
    beforeRowWrite = async () => {};
    reached.resolve();
    await commit.promise;
  };
  return { reached: reached.promise, commit: commit.resolve };
}

/** Request a cut and report whether it was granted yet. */
function requestCut() {
  const state = { granted: false, release: null };
  state.promise = acquireBackupSnapshotCut().then((release) => {
    state.granted = true;
    state.release = release;
    return release;
  });
  return state;
}

let seq = 0;
const newId = () => `sprite-${++seq}`;

async function lockedCharacter(directions = ['east']) {
  const id = newId();
  await records.createRecord({ kind: 'character', name: 'Example' }, id);
  await lockAllAnchors(TEST_ROOT, id, { lockReference, directions, records });
  return id;
}

async function makeCandidateRun(recordId, direction) {
  const runId = `walk-${direction}-${(seq++).toString(16).padStart(8, '0')}`;
  const runDir = join(TEST_ROOT, 'sprites', recordId, 'runs', runId, 'generated');
  await mkdir(runDir, { recursive: true });
  const stripName = `${recordId}-walk-${direction}-strip.png`;
  await writeFile(join(runDir, stripName), `strip-${direction}`);
  const manifestRel = `runs/${runId}/generated/${recordId}-walk-${direction}-manifest.json`;
  await writeFile(spritePath(recordId, manifestRel), JSON.stringify({
    schemaVersion: 1,
    kind: 'deterministically-packaged-grok-walk-video',
    characterId: recordId,
    direction,
    frameRate: 12,
    frameCount: 8,
    stripPath: `runs/${runId}/generated/${stripName}`,
    stripSha256: createHash('sha256').update(Buffer.from(`strip-${direction}`)).digest('hex'),
  }));
  await writeFile(spritePath(recordId, `runs/${runId}/animation-run.json`), JSON.stringify({
    schemaVersion: 1,
    kind: 'grok-game-animation-frames-run',
    status: 'candidate',
    id: runId,
    characterId: recordId,
    direction,
    chromaKey: '#FF00FF',
    createdAt: new Date().toISOString(),
    postprocessManifest: manifestRel,
    sourceVideoPath: `runs/${runId}/generated/source-video.mp4`,
    stripPreview: { stripPath: `runs/${runId}/generated/${stripName}`, frameCount: 8, fps: 12, cellWidth: 384, cellHeight: 384, row: 0, startColumn: 0 },
  }));
  return runId;
}

const approveDirections = async (id, directions) => {
  for (const direction of directions) {
    // eslint-disable-next-line no-await-in-loop -- each approval rewrites the shared selection file
    await approveWalkDirection(id, { direction, runId: await makeCandidateRun(id, direction) });
  }
};

async function finalizedCharacter() {
  const id = await lockedCharacter(ANCHOR_DIRECTIONS);
  await approveDirections(id, SPRITE_DIRECTIONS);
  expect((await records.getRecord(id)).status).toBe('walk-complete');
  return id;
}

beforeEach(() => {
  beforeRowWrite = async () => {};
  afterRunWrite = async () => {};
  executeTuiRun.mockReset();
  executeTuiRun.mockImplementation(() => new Promise(() => {}));
  runWalkPostprocess.mockClear();
});

/**
 * One in-flight workflow, parked after its files changed and before its row
 * commits: a cut requested now must wait for the workflow and only then be
 * granted, so the dump that follows the file copy sees the finished pair.
 */
async function expectCutDrainsWorkflow({ start, filesChanged, committed }) {
  const hold = holdRowWrite();
  const running = start();
  running.catch(() => {}); // surfaced by the await below; never left unhandled if an assertion throws first
  let cut;
  try {
    await Promise.race([hold.reached, running.then(() => { throw new Error('workflow ended before its row write'); })]);
    await filesChanged();
    cut = requestCut();
    await settle();
    expect(cut.granted).toBe(false);
    hold.commit();
    // A workflow may take a second lease after the first (a revision, then the
    // unlock); it is admitted again only once the cut is released.
    await cut.promise;
    await committed();
  } finally {
    hold.commit();
    cut?.promise.then((release) => release(), () => {});
  }
  await running;
}

/** A workflow requested during a cut must change no byte until the cut is released. */
async function expectWorkflowWaitsOutCut({ start, untouched, applied }) {
  const release = await acquireBackupSnapshotCut();
  let running;
  try {
    running = start();
    running.catch(() => {});
    await settle();
    await untouched();
  } finally {
    release();
  }
  await running;
  await applied();
}

describe('reference lock and unlock', () => {
  it('drains an in-flight turnaround lock before a cut is granted', async () => {
    const id = newId();
    await records.createRecord({ kind: 'character', name: 'Example' }, id);
    const candidate = await placeCandidateFixture(TEST_ROOT, id, 'turnaround', 'turnaround-candidate-01.png');
    await expectCutDrainsWorkflow({
      start: () => lockReference(id, { target: 'turnaround', candidate }),
      // The lock writes the frozen sheet first; the manifest names it only after the row.
      filesChanged: async () => {
        expect(await exists(spritePath(id, `reference/${id}-turnaround-v1.png`))).toBe(true);
        expect(await exists(spritePath(id, `reference/${id}-reference-set-v1.json`))).toBe(false);
      },
      committed: async () => {
        const manifest = await readJson(spritePath(id, `reference/${id}-reference-set-v1.json`));
        expect(manifest.turnaround.locked).toBe(true);
        expect((await records.getRecord(id)).status).toBe('reference');
      },
    });
  });

  it('writes no locked sheet while a cut is active', async () => {
    const id = newId();
    await records.createRecord({ kind: 'character', name: 'Example' }, id);
    const candidate = await placeCandidateFixture(TEST_ROOT, id, 'turnaround', 'turnaround-candidate-01.png');
    await expectWorkflowWaitsOutCut({
      start: () => lockReference(id, { target: 'turnaround', candidate }),
      untouched: async () => {
        expect(await exists(spritePath(id, `reference/${id}-turnaround-v1.png`))).toBe(false);
        expect((await records.getRecord(id)).status).toBe('draft');
      },
      applied: async () => {
        expect(await exists(spritePath(id, `reference/${id}-turnaround-v1.png`))).toBe(true);
        expect((await records.getRecord(id)).status).toBe('reference');
      },
    });
  });

  // The unlocks write the row first, then the manifest, so a cut parked on the row
  // still sees a locked manifest; the lease keeps the dump from pairing that
  // manifest with the already-downgraded row.
  it.each([
    ['anchor', (id) => unlockReferenceAnchor(id, { direction: 'east' }), (m) => m.anchors.find((a) => a.direction === 'east').status],
    ['main', (id) => unlockReferenceMain(id), (m) => (m.mainReference.locked ? 'locked' : 'pending')],
    ['turnaround', (id) => unlockReferenceTurnaround(id), (m) => (m.turnaround.locked ? 'locked' : 'pending')],
  ])('keeps a %s unlock whole across a cut', async (_name, unlock, manifestState) => {
    const id = await lockedCharacter(['east']);
    const manifestPath = (recordId) => spritePath(recordId, `reference/${recordId}-reference-set-v1.json`);
    const before = (await records.getRecord(id)).updatedAt;
    await new Promise((resolve) => setTimeout(resolve, 5)); // a distinct updatedAt millisecond
    await expectCutDrainsWorkflow({
      start: () => unlock(id),
      filesChanged: async () => expect(manifestState(await readJson(manifestPath(id)))).toBe('locked'),
      committed: async () => {
        expect(manifestState(await readJson(manifestPath(id)))).toBe('pending');
        expect((await records.getRecord(id)).updatedAt).not.toBe(before);
      },
    });
    const second = await lockedCharacter(['east']);
    const secondBefore = (await records.getRecord(second)).updatedAt;
    await expectWorkflowWaitsOutCut({
      start: () => unlock(second),
      untouched: async () => {
        expect(manifestState(await readJson(manifestPath(second)))).toBe('locked');
        expect((await records.getRecord(second)).updatedAt).toBe(secondBefore);
      },
      applied: async () => expect(manifestState(await readJson(manifestPath(second)))).toBe('pending'),
    });
  });
});

describe('walk set finalization and revision', () => {
  // Finalizing writes the walk set before the row, so a snapshot could pair a copy
  // without the set with a row that says walk-complete — the state its write order
  // exists to prevent.
  it('keeps the eighth approval whole across a cut', async () => {
    const id = await lockedCharacter(ANCHOR_DIRECTIONS);
    await approveDirections(id, SPRITE_DIRECTIONS.slice(0, 7));
    const last = SPRITE_DIRECTIONS[7];
    const runId = await makeCandidateRun(id, last);
    const walkSet = spritePath(id, `walk/${id}-walk-set-v1.json`);
    await expectCutDrainsWorkflow({
      start: () => approveWalkDirection(id, { direction: last, runId }),
      filesChanged: async () => expect(await exists(walkSet)).toBe(true),
      committed: async () => expect((await records.getRecord(id)).status).toBe('walk-complete'),
    });
  });

  it('writes no walk set while a cut is active', async () => {
    const id = await lockedCharacter(ANCHOR_DIRECTIONS);
    await approveDirections(id, SPRITE_DIRECTIONS.slice(0, 7));
    const last = SPRITE_DIRECTIONS[7];
    const runId = await makeCandidateRun(id, last);
    const walkSet = spritePath(id, `walk/${id}-walk-set-v1.json`);
    await expectWorkflowWaitsOutCut({
      start: () => approveWalkDirection(id, { direction: last, runId }),
      untouched: async () => {
        expect(await exists(walkSet)).toBe(false);
        expect((await records.getRecord(id)).status).toBe('reference-complete');
      },
      applied: async () => expect(await exists(walkSet)).toBe(true),
    });
  });

  // Each of these removes the finalized set and then downgrades the row: a snapshot
  // that copied the set earlier must not dump the downgraded row.
  it.each([
    ['unlockWalkSet', (id) => unlockWalkSet(id, { acknowledgeNoClips: true })],
    ['reopenWalkDirection', (id) => reopenWalkDirection(id, { direction: 'east', acknowledgeNoClips: true })],
    ['invalidateWalkDirectionForAnchorRevision', (id) => invalidateWalkDirectionForAnchorRevision(id, { direction: 'east' })],
    ['unlockTurnaroundReference', (id) => unlockTurnaroundReference(id)],
  ])('keeps %s whole across a cut', async (_name, workflow) => {
    const id = await finalizedCharacter();
    const walkSet = spritePath(id, `walk/${id}-walk-set-v1.json`);
    await expectCutDrainsWorkflow({
      start: () => workflow(id),
      filesChanged: async () => expect(await exists(walkSet)).toBe(false),
      committed: async () => expect((await records.getRecord(id)).status).not.toBe('walk-complete'),
    });
    const second = await finalizedCharacter();
    await expectWorkflowWaitsOutCut({
      start: () => workflow(second),
      untouched: async () => {
        expect(await exists(spritePath(second, `walk/${second}-walk-set-v1.json`))).toBe(true);
        expect((await records.getRecord(second)).status).toBe('walk-complete');
      },
      applied: async () => expect(await exists(spritePath(second, `walk/${second}-walk-set-v1.json`))).toBe(false),
    });
  });
});

describe('grok TUI attach lane', () => {
  // The terminal session runs outside admission and writes the clip itself; the
  // attach that packages its frames and files the run record is the publication.
  it.each([
    ['walk', (id) => startWalkGeneration(id, { direction: 'east' })],
    ['scanner track', (id) => startTrackGeneration('scanner', id, { direction: 'east' })],
  ])('packages a %s clip and files its run only outside a cut', async (_name, start) => {
    const id = await lockedCharacter(['east']);
    const clipWritten = deferred();
    const sessionEnds = deferred();
    executeTuiRun.mockImplementationOnce(async ({ workspacePath }) => {
      await writeFile(join(workspacePath, 'source-video.mp4'), 'grok-clip-bytes');
      clipWritten.resolve();
      await sessionEnds.promise;
    });
    const { runId } = await start(id);
    await clipWritten.promise;
    const runStatus = async () => (await readJson(spritePath(id, `runs/${runId}/animation-run.json`))).status;

    const terminalWrite = deferred();
    afterRunWrite = async (path, content) => {
      if (path === spritePath(id, `runs/${runId}/animation-run.json`) &&
          ['candidate', 'error'].includes((typeof content === 'string' ? JSON.parse(content) : content).status)) terminalWrite.resolve();
    };
    const release = await acquireBackupSnapshotCut();
    try {
      sessionEnds.resolve();
      await settle();
      expect(runWalkPostprocess).not.toHaveBeenCalled();
      expect(await runStatus()).toBe('rendering');
    } finally {
      release();
    }
    // Join the durable terminal write rather than racing filesystem work against
    // vi.waitFor's polling timeout. The next cut also joins the attachment's
    // publication finalizer before this fixture can be torn down.
    await terminalWrite.promise;
    const releaseSettledCut = await acquireBackupSnapshotCut();
    try {
      expect(await runStatus()).toBe('candidate');
      expect(runWalkPostprocess).toHaveBeenCalledOnce();
    } finally {
      releaseSettledCut();
      afterRunWrite = async () => {};
    }
  });
});

describe('source-pipeline import', () => {
  const sourceRoot = () => join(TEST_ROOT, 'source-pipeline');
  // A character subject carries a spec; a props family is a directory of atlas images.
  const SUBJECTS = [
    {
      name: 'character',
      async write(id) {
        await mkdir(join(sourceRoot(), 'art-pipeline', 'characters'), { recursive: true });
        await writeFile(
          join(sourceRoot(), 'art-pipeline', 'characters', `${id}.json`),
          JSON.stringify({ characterId: id, displayName: 'Example Importee' }),
        );
      },
      imported: (id) => spritePath(id, 'character-spec.json'),
    },
    {
      name: 'props family',
      async write(id) {
        await mkdir(join(sourceRoot(), 'game', 'assets', 'sprites', id), { recursive: true });
        await writeFile(join(sourceRoot(), 'game', 'assets', 'sprites', id, 'props.png'), 'synthetic atlas');
      },
      imported: (id) => spritePath(id, 'atlas/props.png'),
    },
  ];
  const run = (id) => importFromSource({ sourceRoot: sourceRoot(), characters: [id], includeProps: true });

  it.each(SUBJECTS)('drains an in-flight $name import before a cut is granted', async ({ write, imported }) => {
    const id = newId();
    await write(id);
    await expectCutDrainsWorkflow({
      start: () => run(id),
      filesChanged: async () => expect(await exists(imported(id))).toBe(true),
      committed: async () => expect((await records.getRecord(id)).status).toBe('imported'),
    });
  });

  it.each(SUBJECTS)('copies no $name file while a cut is active', async ({ write, imported }) => {
    const id = newId();
    await write(id);
    await expectWorkflowWaitsOutCut({
      start: () => run(id),
      untouched: async () => {
        expect(await exists(imported(id))).toBe(false);
        expect(await records.getRecord(id)).toBeNull();
      },
      applied: async () => expect((await records.getRecord(id)).status).toBe('imported'),
    });
  });
});

// The DB sprite row remains metadata-only. Adjacent file-primary records still
// name bytes and are separately admitted by the shared animation/reference tails.
describe('sprite row contents', () => {
  it('holds metadata and workflow state only, never a data/ path', () => {
    const record = buildSpriteRecord({ name: 'Example', kind: 'character' }, { id: 'example', now: '2026-01-01T00:00:00.000Z' });
    expect(Object.keys(record).sort()).toEqual([
      'chromaKey', 'createdAt', 'deleted', 'deletedAt', 'id', 'imageMode', 'imageModelId', 'importedFrom',
      'kind', 'name', 'notes', 'publishBinding', 'spec', 'status', 'updatedAt',
    ]);
  });
});