Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
const failures = vi.hoisted(() => ({ write: false, unsettled: vi.fn() }));
vi.mock('../../lib/maintenanceAdmission.js', () => ({ maintenance: { run: (_kind, _resource, fn) => fn(), markCurrentUnsettled: failures.unsettled } }));
vi.mock('../../lib/fileUtils.js', async (load) => {
const actual = await load();
return { ...actual, atomicWrite: async (path, body) => {
if (failures.write) {
const { writeFile } = await import('node:fs/promises');
await writeFile(`${path}.fixture.tmp`, body.subarray(0, 4));
throw Object.assign(new Error('Fixture partial persistence failure'), { code: 'EIO' });
}
return actual.atomicWrite(path, body);
} };
});
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { mkdtempSync, rmSync, writeFileSync, existsSync, utimesSync, truncateSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { PATHS } from '../../lib/fileUtils.js';
import {
FEDERATED_MEDIA_ASSET_TTL_MS,
federatedMediaAssetId,
federatedMediaAssetOwner,
} from '../../lib/federatedMediaWire.js';
import {
FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES,
FEDERATED_MEDIA_ASSET_CALLER_MAX_COUNT,
FEDERATED_MEDIA_ASSET_TOTAL_MAX_BYTES,
__resetFederatedAssetSweepForTests,
collectActiveFederatedAssetBasenames,
describeFederatedMediaAsset,
findFederatedMediaAsset,
storeFederatedMediaAsset,
sweepFederatedMediaAssets,
} from './assetStore.js';
// Minimal real PNG: the store sniffs magic bytes rather than trusting the
// declared Content-Type, so a placeholder buffer would be refused.
const PNG = Buffer.concat([
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
Buffer.from('portos-test-conditioning-image'),
]);
const JPEG = Buffer.concat([Buffer.from([0xff, 0xd8, 0xff]), Buffer.from('jpeg-body')]);
let tempRoot;
let originalInbox;
beforeEach(() => {
failures.write = false; failures.unsettled.mockClear();
tempRoot = mkdtempSync(join(tmpdir(), 'portos-federated-inbox-'));
originalInbox = PATHS.federatedMediaInbox;
PATHS.federatedMediaInbox = tempRoot;
// The sweep throttles itself to one run per few minutes, which is a
// duplicate-work guard rather than a behavior these cases are asserting.
__resetFederatedAssetSweepForTests();
});
afterEach(() => {
PATHS.federatedMediaInbox = originalInbox;
rmSync(tempRoot, { recursive: true, force: true });
vi.restoreAllMocks();
});
const sha256 = async (buf) => (await import('node:crypto'))
.createHash('sha256').update(buf).digest('hex');
const upload = async (callerId, body = PNG, mimeType = 'image/png', digest) => storeFederatedMediaAsset({
callerId,
mimeType,
declaredSha256: digest ?? await sha256(body),
body,
});
const distinctPng = (n) => Buffer.concat([PNG, Buffer.from(`-variant-${n}`)]);
// Sparse file: reports its logical size to lstat without writing the bytes.
const plantSparse = (callerId, tag, sizeBytes, ageMs = 0) => {
const path = join(tempRoot, `${federatedMediaAssetOwner(callerId)}-${tag.padStart(64, '0')}.png`);
writeFileSync(path, '');
truncateSync(path, sizeBytes);
const when = (Date.now() - ageMs) / 1000;
utimesSync(path, when, when);
return path;
};
describe('federated media asset store', () => {
describe('storage quota', () => {
// Each real upload re-scans the whole inbox and fsyncs a temp+rename write,
// so filling the cap with ~64 of them is quadratic file-system work that
// blew the 30s test deadline on Windows CI (#12701). Stage the bulk as
// count-only placeholders (what the quota scan reads is name + size) and
// let the last slot, the refusal and the replay go through the real path.
const fillCount = (callerId, count) => {
for (let i = 0; i < count; i += 1) plantSparse(callerId, `fill${i}`, 1);
};
it('refuses a caller beyond the per-caller count with 429, but still lets another caller upload and a replay refresh', async () => {
fillCount('peer-a', FEDERATED_MEDIA_ASSET_CALLER_MAX_COUNT - 1);
await upload('peer-a', distinctPng(0));
await expect(upload('peer-a', distinctPng(9999))).rejects.toMatchObject({ status: 429, code: 'MEDIA_PROVIDER_ASSET_QUOTA' });
await expect(upload('peer-a', distinctPng(0))).resolves.toMatchObject({ sizeBytes: distinctPng(0).length });
await expect(upload('peer-b', distinctPng(9999))).resolves.toMatchObject({ assetId: expect.any(String) });
});
it('does not let concurrent uploads jointly overshoot the per-caller count', async () => {
fillCount('peer-a', FEDERATED_MEDIA_ASSET_CALLER_MAX_COUNT - 8);
const results = await Promise.allSettled(
Array.from({ length: 16 }, (_, i) => upload('peer-a', distinctPng(i))),
);
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(8);
expect(results.filter((r) => r.status === 'rejected' && r.reason.code === 'MEDIA_PROVIDER_ASSET_QUOTA')).toHaveLength(8);
});
it('refuses a caller beyond the per-caller byte cap', async () => {
plantSparse('peer-a', 'a', FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES - 10);
await expect(upload('peer-a')).rejects.toMatchObject({ status: 429, code: 'MEDIA_PROVIDER_ASSET_QUOTA' });
});
it('refuses any caller once the store total is full', async () => {
const chunk = FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES - 1;
const chunks = Math.ceil(FEDERATED_MEDIA_ASSET_TOTAL_MAX_BYTES / chunk);
for (let i = 0; i < chunks; i += 1) plantSparse(`other-${i}`, String(i), Math.min(chunk, FEDERATED_MEDIA_ASSET_TOTAL_MAX_BYTES - i * chunk));
await expect(upload('peer-z')).rejects.toMatchObject({ status: 429, code: 'MEDIA_PROVIDER_ASSET_QUOTA' });
});
it('sweeps expired assets on upload before evaluating the cap, but keeps ones a live job pins', async () => {
const stale = plantSparse('peer-a', 'b', FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES - 10, FEDERATED_MEDIA_ASSET_TTL_MS + 60_000);
await expect(upload('peer-a')).resolves.toMatchObject({ sizeBytes: PNG.length });
expect(existsSync(stale)).toBe(false);
const pinned = plantSparse('peer-c', 'c', FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES - 10, FEDERATED_MEDIA_ASSET_TTL_MS + 60_000);
const jobs = [{ status: 'running', params: { initImagePath: pinned } }];
await expect(storeFederatedMediaAsset({
callerId: 'peer-c', mimeType: 'image/png', declaredSha256: await sha256(PNG), body: PNG, jobs,
})).rejects.toMatchObject({ code: 'MEDIA_PROVIDER_ASSET_QUOTA' });
expect(existsSync(pinned)).toBe(true);
});
});
it('keeps input admission unsettled when shared persistence may leave an unknown partial temp file', async () => {
failures.write = true;
await expect(upload('peer-a')).rejects.toMatchObject({ code: 'EIO' });
expect(failures.unsettled).toHaveBeenCalledOnce();
expect(await findFederatedMediaAsset('peer-a', federatedMediaAssetId('peer-a', await sha256(PNG)))).toBeNull();
});
it('accepts a verified image and answers with a content-addressed, caller-scoped id', async () => {
const stored = await upload('peer-a');
expect(stored).toMatchObject({
wireVersion: 1,
sha256: await sha256(PNG),
sizeBytes: PNG.length,
mimeType: 'image/png',
});
expect(stored.assetId).toBe(federatedMediaAssetId('peer-a', await sha256(PNG)));
await expect(findFederatedMediaAsset('peer-a', stored.assetId)).resolves.toMatchObject({
mimeType: 'image/png',
sizeBytes: PNG.length,
});
});
// Content-addressing is what makes a reconcile after a restart cheap: the
// consumer re-sends the same bytes and gets the same slot back rather than a
// second staged copy.
it('is idempotent for identical bytes from the same caller', async () => {
const first = await upload('peer-a');
const second = await upload('peer-a');
expect(second.assetId).toBe(first.assetId);
});
// The caller half of the id is derived from the AUTHENTICATED caller, not
// parsed from the request — so it is an authorization check, not a namespace.
it('refuses to resolve another caller’s asset even given its exact id', async () => {
const stored = await upload('peer-a');
await expect(findFederatedMediaAsset('peer-b', stored.assetId)).resolves.toBeNull();
await expect(describeFederatedMediaAsset('peer-b', stored.assetId))
.rejects.toMatchObject({ status: 404, code: 'MEDIA_PROVIDER_ASSET_NOT_FOUND' });
// Identical bytes from a different caller are a different asset, not a
// shared one — otherwise one peer could observe another peer's uploads by
// guessing content.
const other = await upload('peer-b');
expect(other.assetId).not.toBe(stored.assetId);
});
it('rejects a digest that does not match the bytes rather than staging them', async () => {
await expect(upload('peer-a', PNG, 'image/png', 'f'.repeat(64)))
.rejects.toMatchObject({ code: 'MEDIA_PROVIDER_ASSET_INTEGRITY' });
await expect(sweepFederatedMediaAssets({ force: true })).resolves.toBe(0);
});
// The declared Content-Type is the caller's word for what this is; the magic
// bytes are what the generator will actually open.
it('rejects bytes whose magic number contradicts the declared type', async () => {
await expect(upload('peer-a', JPEG, 'image/png'))
.rejects.toMatchObject({ code: 'MEDIA_PROVIDER_ASSET_TYPE_MISMATCH' });
});
it('rejects a media type outside the conditioning allowlist', async () => {
await expect(upload('peer-a', PNG, 'image/gif'))
.rejects.toMatchObject({ status: 415, code: 'MEDIA_PROVIDER_ASSET_TYPE_UNSUPPORTED' });
});
it('rejects an empty body instead of staging a zero-byte conditioning image', async () => {
await expect(upload('peer-a', Buffer.alloc(0)))
.rejects.toMatchObject({ code: 'MEDIA_PROVIDER_ASSET_EMPTY' });
});
// Staged bytes are another machine's, held to run one job. They expire rather
// than accumulating — and an expired asset must read as absent BEFORE the
// sweep runs, or a job admitted between sweeps would render from bytes the
// TTL had already disowned.
it('treats an expired asset as absent and sweeps it', async () => {
const stored = await upload('peer-a');
const staged = (await findFederatedMediaAsset('peer-a', stored.assetId)).path;
const stale = (Date.now() - FEDERATED_MEDIA_ASSET_TTL_MS - 60_000) / 1000;
utimesSync(staged, stale, stale);
await expect(findFederatedMediaAsset('peer-a', stored.assetId)).resolves.toBeNull();
await expect(sweepFederatedMediaAssets({ force: true })).resolves.toBe(1);
expect(existsSync(staged)).toBe(false);
});
it('leaves a live asset alone when sweeping', async () => {
const stored = await upload('peer-a');
await expect(sweepFederatedMediaAssets({ force: true })).resolves.toBe(0);
await expect(findFederatedMediaAsset('peer-a', stored.assetId)).resolves.not.toBeNull();
});
// The id shape is the containment check: anything that is not
// <16-hex>-<64-hex> cannot address a file, so a traversal attempt never
// reaches the resolver.
it.each([
['a traversal attempt', '../../../etc/passwd'],
['an absolute path', '/etc/passwd'],
['a bare filename', 'photo.png'],
['a malformed pair', `${'0'.repeat(15)}-${'a'.repeat(64)}`],
])('refuses %s as an asset id', async (_name, assetId) => {
await expect(findFederatedMediaAsset('peer-a', assetId)).resolves.toBeNull();
});
// A file dropped into the inbox by hand — the sweep and the resolver both key
// on the id shape, so an unparseable name is unreachable rather than a path
// the generator could be pointed at.
it('ignores a hand-written file whose name is not a valid asset id', async () => {
writeFileSync(join(tempRoot, 'not-an-asset.png'), PNG);
await expect(findFederatedMediaAsset('peer-a', 'not-an-asset')).resolves.toBeNull();
});
it('answers a missing inbox directory with zero swept rather than throwing', async () => {
rmSync(tempRoot, { recursive: true, force: true });
await expect(sweepFederatedMediaAssets({ force: true })).resolves.toBe(0);
});
// The age gate is a BACKSTOP, not the safety property — the lesson
// imageCleanTmpGc.js records for the structurally identical image-clean-tmp
// dir. A federated job queued behind a long render or a first-run model
// download can easily sit past the TTL, and deleting its staged init image
// leaves the runner unable to open a file the consumer already committed to
// and is waiting on. Nothing re-uploads it: the ids were resolved at
// admission, and the provider never tells the consumer the bytes went away.
describe('active-job pinning', () => {
const expire = (path) => {
const stale = (Date.now() - FEDERATED_MEDIA_ASSET_TTL_MS - 60_000) / 1000;
utimesSync(path, stale, stale);
};
it('spares an expired asset a queued job still depends on', async () => {
const stored = await upload('peer-a');
const staged = (await findFederatedMediaAsset('peer-a', stored.assetId)).path;
expire(staged);
const jobs = [{ status: 'queued', params: { initImagePath: staged } }];
await expect(sweepFederatedMediaAssets({ jobs, force: true })).resolves.toBe(0);
expect(existsSync(staged)).toBe(true);
});
it('collects every conditioning slot, from paths and from arrays alike', () => {
const keep = collectActiveFederatedAssetBasenames([
{ status: 'running', params: { initImagePath: '/inbox/a.png' } },
{ status: 'queued', params: { referenceImagePaths: ['/inbox/b.png', '/inbox/c.png'] } },
{ status: 'queued', params: { sourceImagePath: '/inbox/d.png', lastImagePath: '/inbox/e.png' } },
]);
expect([...keep].sort()).toEqual(['a.png', 'b.png', 'c.png', 'd.png', 'e.png']);
});
// A terminal job has already rendered, so its conditioning is free to go —
// otherwise the inbox would only ever grow.
it.each([['completed'], ['failed'], ['canceled']])('does not pin a %s job', async (status) => {
const stored = await upload('peer-a');
const staged = (await findFederatedMediaAsset('peer-a', stored.assetId)).path;
expire(staged);
const jobs = [{ status, params: { initImagePath: staged } }];
await expect(sweepFederatedMediaAssets({ jobs, force: true })).resolves.toBe(1);
expect(existsSync(staged)).toBe(false);
});
it('tolerates jobs with no conditioning params at all', () => {
expect(collectActiveFederatedAssetBasenames([
{ status: 'queued', params: { prompt: 'a lighthouse' } },
{ status: 'running' },
null,
]).size).toBe(0);
expect(collectActiveFederatedAssetBasenames().size).toBe(0);
});
});
});