Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

14.5 KB
Raw Blame
const failures = vi.hoisted(() => ({ write: false, unsettled: vi.fn() }));
vi.mock('../../lib/maintenanceAdmission.js', () => ({ maintenance: { run: (_kind, _resource, fn) => fn(), markCurrentUnsettled: failures.unsettled } }));
vi.mock('../../lib/fileUtils.js', async (load) => {
  const actual = await load();
  return { ...actual, atomicWrite: async (path, body) => {
    if (failures.write) {
      const { writeFile } = await import('node:fs/promises');
      await writeFile(`${path}.fixture.tmp`, body.subarray(0, 4));
      throw Object.assign(new Error('Fixture partial persistence failure'), { code: 'EIO' });
    }
    return actual.atomicWrite(path, body);
  } };
});
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { mkdtempSync, rmSync, writeFileSync, existsSync, utimesSync, truncateSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { PATHS } from '../../lib/fileUtils.js';
import {
  FEDERATED_MEDIA_ASSET_TTL_MS,
  federatedMediaAssetId,
  federatedMediaAssetOwner,
} from '../../lib/federatedMediaWire.js';
import {
  FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES,
  FEDERATED_MEDIA_ASSET_CALLER_MAX_COUNT,
  FEDERATED_MEDIA_ASSET_TOTAL_MAX_BYTES,
  __resetFederatedAssetSweepForTests,
  collectActiveFederatedAssetBasenames,
  describeFederatedMediaAsset,
  findFederatedMediaAsset,
  storeFederatedMediaAsset,
  sweepFederatedMediaAssets,
} from './assetStore.js';

// Minimal real PNG: the store sniffs magic bytes rather than trusting the
// declared Content-Type, so a placeholder buffer would be refused.
const PNG = Buffer.concat([
  Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
  Buffer.from('portos-test-conditioning-image'),
]);
const JPEG = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff]), Buffer.from('jpeg-body')]);

let tempRoot;
let originalInbox;

beforeEach(() => {
  failures.write = false; failures.unsettled.mockClear();
  tempRoot = mkdtempSync(join(tmpdir(), 'portos-federated-inbox-'));
  originalInbox = PATHS.federatedMediaInbox;
  PATHS.federatedMediaInbox = tempRoot;
  // The sweep throttles itself to one run per few minutes, which is a
  // duplicate-work guard rather than a behavior these cases are asserting.
  __resetFederatedAssetSweepForTests();
});

afterEach(() => {
  PATHS.federatedMediaInbox = originalInbox;
  rmSync(tempRoot, { recursive: true, force: true });
  vi.restoreAllMocks();
});

const sha256 = async (buf) => (await import('node:crypto'))
  .createHash('sha256').update(buf).digest('hex');

const upload = async (callerId, body = PNG, mimeType = 'image/png', digest) => storeFederatedMediaAsset({
  callerId,
  mimeType,
  declaredSha256: digest ?? await sha256(body),
  body,
});

const distinctPng = (n) => Buffer.concat([PNG, Buffer.from(`-variant-${n}`)]);

// Sparse file: reports its logical size to lstat without writing the bytes.
const plantSparse = (callerId, tag, sizeBytes, ageMs = 0) => {
  const path = join(tempRoot, `${federatedMediaAssetOwner(callerId)}-${tag.padStart(64, '0')}.png`);
  writeFileSync(path, '');
  truncateSync(path, sizeBytes);
  const when = (Date.now() - ageMs) / 1000;
  utimesSync(path, when, when);
  return path;
};

describe('federated media asset store', () => {
  describe('storage quota', () => {
    // Each real upload re-scans the whole inbox and fsyncs a temp+rename write,
    // so filling the cap with ~64 of them is quadratic file-system work that
    // blew the 30s test deadline on Windows CI (#12701). Stage the bulk as
    // count-only placeholders (what the quota scan reads is name + size) and
    // let the last slot, the refusal and the replay go through the real path.
    const fillCount = (callerId, count) => {
      for (let i = 0; i < count; i += 1) plantSparse(callerId, `fill${i}`, 1);
    };

    it('refuses a caller beyond the per-caller count with 429, but still lets another caller upload and a replay refresh', async () => {
      fillCount('peer-a', FEDERATED_MEDIA_ASSET_CALLER_MAX_COUNT - 1);
      await upload('peer-a', distinctPng(0));
      await expect(upload('peer-a', distinctPng(9999))).rejects.toMatchObject({ status: 429, code: 'MEDIA_PROVIDER_ASSET_QUOTA' });
      await expect(upload('peer-a', distinctPng(0))).resolves.toMatchObject({ sizeBytes: distinctPng(0).length });
      await expect(upload('peer-b', distinctPng(9999))).resolves.toMatchObject({ assetId: expect.any(String) });
    });

    it('does not let concurrent uploads jointly overshoot the per-caller count', async () => {
      fillCount('peer-a', FEDERATED_MEDIA_ASSET_CALLER_MAX_COUNT - 8);
      const results = await Promise.allSettled(
        Array.from({ length: 16 }, (_, i) => upload('peer-a', distinctPng(i))),
      );
      expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(8);
      expect(results.filter((r) => r.status === 'rejected' && r.reason.code === 'MEDIA_PROVIDER_ASSET_QUOTA')).toHaveLength(8);
    });

    it('refuses a caller beyond the per-caller byte cap', async () => {
      plantSparse('peer-a', 'a', FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES - 10);
      await expect(upload('peer-a')).rejects.toMatchObject({ status: 429, code: 'MEDIA_PROVIDER_ASSET_QUOTA' });
    });

    it('refuses any caller once the store total is full', async () => {
      const chunk = FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES - 1;
      const chunks = Math.ceil(FEDERATED_MEDIA_ASSET_TOTAL_MAX_BYTES / chunk);
      for (let i = 0; i < chunks; i += 1) plantSparse(`other-${i}`, String(i), Math.min(chunk, FEDERATED_MEDIA_ASSET_TOTAL_MAX_BYTES - i * chunk));
      await expect(upload('peer-z')).rejects.toMatchObject({ status: 429, code: 'MEDIA_PROVIDER_ASSET_QUOTA' });
    });

    it('sweeps expired assets on upload before evaluating the cap, but keeps ones a live job pins', async () => {
      const stale = plantSparse('peer-a', 'b', FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES - 10, FEDERATED_MEDIA_ASSET_TTL_MS + 60_000);
      await expect(upload('peer-a')).resolves.toMatchObject({ sizeBytes: PNG.length });
      expect(existsSync(stale)).toBe(false);

      const pinned = plantSparse('peer-c', 'c', FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES - 10, FEDERATED_MEDIA_ASSET_TTL_MS + 60_000);
      const jobs = [{ status: 'running', params: { initImagePath: pinned } }];
      await expect(storeFederatedMediaAsset({
        callerId: 'peer-c', mimeType: 'image/png', declaredSha256: await sha256(PNG), body: PNG, jobs,
      })).rejects.toMatchObject({ code: 'MEDIA_PROVIDER_ASSET_QUOTA' });
      expect(existsSync(pinned)).toBe(true);
    });
  });

  it('keeps input admission unsettled when shared persistence may leave an unknown partial temp file', async () => {
    failures.write = true;
    await expect(upload('peer-a')).rejects.toMatchObject({ code: 'EIO' });
    expect(failures.unsettled).toHaveBeenCalledOnce();
    expect(await findFederatedMediaAsset('peer-a', federatedMediaAssetId('peer-a', await sha256(PNG)))).toBeNull();
  });
  it('accepts a verified image and answers with a content-addressed, caller-scoped id', async () => {
    const stored = await upload('peer-a');
    expect(stored).toMatchObject({
      wireVersion: 1,
      sha256: await sha256(PNG),
      sizeBytes: PNG.length,
      mimeType: 'image/png',
    });
    expect(stored.assetId).toBe(federatedMediaAssetId('peer-a', await sha256(PNG)));
    await expect(findFederatedMediaAsset('peer-a', stored.assetId)).resolves.toMatchObject({
      mimeType: 'image/png',
      sizeBytes: PNG.length,
    });
  });

  // Content-addressing is what makes a reconcile after a restart cheap: the
  // consumer re-sends the same bytes and gets the same slot back rather than a
  // second staged copy.
  it('is idempotent for identical bytes from the same caller', async () => {
    const first = await upload('peer-a');
    const second = await upload('peer-a');
    expect(second.assetId).toBe(first.assetId);
  });

  // The caller half of the id is derived from the AUTHENTICATED caller, not
  // parsed from the request — so it is an authorization check, not a namespace.
  it('refuses to resolve another caller’s asset even given its exact id', async () => {
    const stored = await upload('peer-a');
    await expect(findFederatedMediaAsset('peer-b', stored.assetId)).resolves.toBeNull();
    await expect(describeFederatedMediaAsset('peer-b', stored.assetId))
      .rejects.toMatchObject({ status: 404, code: 'MEDIA_PROVIDER_ASSET_NOT_FOUND' });
    // Identical bytes from a different caller are a different asset, not a
    // shared one — otherwise one peer could observe another peer's uploads by
    // guessing content.
    const other = await upload('peer-b');
    expect(other.assetId).not.toBe(stored.assetId);
  });

  it('rejects a digest that does not match the bytes rather than staging them', async () => {
    await expect(upload('peer-a', PNG, 'image/png', 'f'.repeat(64)))
      .rejects.toMatchObject({ code: 'MEDIA_PROVIDER_ASSET_INTEGRITY' });
    await expect(sweepFederatedMediaAssets({ force: true })).resolves.toBe(0);
  });

  // The declared Content-Type is the caller's word for what this is; the magic
  // bytes are what the generator will actually open.
  it('rejects bytes whose magic number contradicts the declared type', async () => {
    await expect(upload('peer-a', JPEG, 'image/png'))
      .rejects.toMatchObject({ code: 'MEDIA_PROVIDER_ASSET_TYPE_MISMATCH' });
  });

  it('rejects a media type outside the conditioning allowlist', async () => {
    await expect(upload('peer-a', PNG, 'image/gif'))
      .rejects.toMatchObject({ status: 415, code: 'MEDIA_PROVIDER_ASSET_TYPE_UNSUPPORTED' });
  });

  it('rejects an empty body instead of staging a zero-byte conditioning image', async () => {
    await expect(upload('peer-a', Buffer.alloc(0)))
      .rejects.toMatchObject({ code: 'MEDIA_PROVIDER_ASSET_EMPTY' });
  });

  // Staged bytes are another machine's, held to run one job. They expire rather
  // than accumulating — and an expired asset must read as absent BEFORE the
  // sweep runs, or a job admitted between sweeps would render from bytes the
  // TTL had already disowned.
  it('treats an expired asset as absent and sweeps it', async () => {
    const stored = await upload('peer-a');
    const staged = (await findFederatedMediaAsset('peer-a', stored.assetId)).path;
    const stale = (Date.now() - FEDERATED_MEDIA_ASSET_TTL_MS - 60_000) / 1000;
    utimesSync(staged, stale, stale);

    await expect(findFederatedMediaAsset('peer-a', stored.assetId)).resolves.toBeNull();
    await expect(sweepFederatedMediaAssets({ force: true })).resolves.toBe(1);
    expect(existsSync(staged)).toBe(false);
  });

  it('leaves a live asset alone when sweeping', async () => {
    const stored = await upload('peer-a');
    await expect(sweepFederatedMediaAssets({ force: true })).resolves.toBe(0);
    await expect(findFederatedMediaAsset('peer-a', stored.assetId)).resolves.not.toBeNull();
  });

  // The id shape is the containment check: anything that is not
  // <16-hex>-<64-hex> cannot address a file, so a traversal attempt never
  // reaches the resolver.
  it.each([
    ['a traversal attempt', '../../../etc/passwd'],
    ['an absolute path', '/etc/passwd'],
    ['a bare filename', 'photo.png'],
    ['a malformed pair', `${'0'.repeat(15)}-${'a'.repeat(64)}`],
  ])('refuses %s as an asset id', async (_name, assetId) => {
    await expect(findFederatedMediaAsset('peer-a', assetId)).resolves.toBeNull();
  });

  // A file dropped into the inbox by hand — the sweep and the resolver both key
  // on the id shape, so an unparseable name is unreachable rather than a path
  // the generator could be pointed at.
  it('ignores a hand-written file whose name is not a valid asset id', async () => {
    writeFileSync(join(tempRoot, 'not-an-asset.png'), PNG);
    await expect(findFederatedMediaAsset('peer-a', 'not-an-asset')).resolves.toBeNull();
  });

  it('answers a missing inbox directory with zero swept rather than throwing', async () => {
    rmSync(tempRoot, { recursive: true, force: true });
    await expect(sweepFederatedMediaAssets({ force: true })).resolves.toBe(0);
  });

  // The age gate is a BACKSTOP, not the safety property — the lesson
  // imageCleanTmpGc.js records for the structurally identical image-clean-tmp
  // dir. A federated job queued behind a long render or a first-run model
  // download can easily sit past the TTL, and deleting its staged init image
  // leaves the runner unable to open a file the consumer already committed to
  // and is waiting on. Nothing re-uploads it: the ids were resolved at
  // admission, and the provider never tells the consumer the bytes went away.
  describe('active-job pinning', () => {
    const expire = (path) => {
      const stale = (Date.now() - FEDERATED_MEDIA_ASSET_TTL_MS - 60_000) / 1000;
      utimesSync(path, stale, stale);
    };

    it('spares an expired asset a queued job still depends on', async () => {
      const stored = await upload('peer-a');
      const staged = (await findFederatedMediaAsset('peer-a', stored.assetId)).path;
      expire(staged);

      const jobs = [{ status: 'queued', params: { initImagePath: staged } }];
      await expect(sweepFederatedMediaAssets({ jobs, force: true })).resolves.toBe(0);
      expect(existsSync(staged)).toBe(true);
    });

    it('collects every conditioning slot, from paths and from arrays alike', () => {
      const keep = collectActiveFederatedAssetBasenames([
        { status: 'running', params: { initImagePath: '/inbox/a.png' } },
        { status: 'queued', params: { referenceImagePaths: ['/inbox/b.png', '/inbox/c.png'] } },
        { status: 'queued', params: { sourceImagePath: '/inbox/d.png', lastImagePath: '/inbox/e.png' } },
      ]);
      expect([...keep].sort()).toEqual(['a.png', 'b.png', 'c.png', 'd.png', 'e.png']);
    });

    // A terminal job has already rendered, so its conditioning is free to go —
    // otherwise the inbox would only ever grow.
    it.each([['completed'], ['failed'], ['canceled']])('does not pin a %s job', async (status) => {
      const stored = await upload('peer-a');
      const staged = (await findFederatedMediaAsset('peer-a', stored.assetId)).path;
      expire(staged);

      const jobs = [{ status, params: { initImagePath: staged } }];
      await expect(sweepFederatedMediaAssets({ jobs, force: true })).resolves.toBe(1);
      expect(existsSync(staged)).toBe(false);
    });

    it('tolerates jobs with no conditioning params at all', () => {
      expect(collectActiveFederatedAssetBasenames([
        { status: 'queued', params: { prompt: 'a lighthouse' } },
        { status: 'running' },
        null,
      ]).size).toBe(0);
      expect(collectActiveFederatedAssetBasenames().size).toBe(0);
    });
  });
});