Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: bound federated media input asset store per caller and in total (#12492)
[._.]/ Adam Eivy0d7954ccee4b
diff --git a/server/routes/federatedMedia.js b/server/routes/federatedMedia.jsindex 5df419566e..29ecc13b70 100644--- a/server/routes/federatedMedia.js+++ b/server/routes/federatedMedia.js@@ -27,6 +27,7 @@ import {getFederatedMediaResult,submitFederatedMediaJob,} from '../services/federatedMediaProvider.js';+import { listJobs } from '../services/mediaJobQueue/index.js';import {describeFederatedMediaAsset,storeFederatedMediaAsset,@@ -73,6 +74,7 @@ router.post('/assets', assetAuth, assetBody, asyncHandler(async (req, res) => {mimeType: (req.get('Content-Type') || '').split(';')[0].trim().toLowerCase(),declaredSha256: req.get('X-Content-SHA256'),body: req.body,+ jobs: listJobs(),});res.status(201).json(stored);}));diff --git a/server/services/federatedMedia/assetStore.js b/server/services/federatedMedia/assetStore.jsindex 5b6154c990..e4dc2daeb7 100644--- a/server/services/federatedMedia/assetStore.js+++ b/server/services/federatedMedia/assetStore.js@@ -44,6 +44,15 @@ import {federatedMediaAssetOwner,} from '../../lib/federatedMediaWire.js';+// Bounds on what an admitted peer can keep staged. The TTL only limits how+// long bytes live, not how many arrive within it, so without these an uploader+// that never submits a job can fill the volume. Per-caller keeps one peer from+// crowding out the others; the total bounds the directory regardless of how+// many peers are admitted.+export const FEDERATED_MEDIA_ASSET_CALLER_MAX_COUNT = 64;+export const FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES = 256 * 1024 * 1024;+export const FEDERATED_MEDIA_ASSET_TOTAL_MAX_BYTES = 1024 * 1024 * 1024;+const reject = (message, code, status = 400, context) => {throw new ServerError(message, { status, code, ...(context ? { context } : {}) });};@@ -110,10 +119,11 @@ export async function findFederatedMediaAsset(callerId, assetId, { now = Date.no* @param {string} args.mimeType - Declared Content-Type.* @param {string} args.declaredSha256 - Caller's X-Content-SHA256 header.* @param {Buffer} args.body+ * @param {object[]} [args.jobs] - live queue jobs, so the upload-time sweep keeps pinned inputs*/export const storeFederatedMediaAsset = (args) => maintenance.run('media-input', 'Federated input upload', () => storeAsset(args));-async function storeAsset({ callerId, mimeType, declaredSha256, body }) {+async function storeAsset({ callerId, mimeType, declaredSha256, body, jobs = [] }) {if (!FEDERATED_MEDIA_ASSET_MIME_TYPES.includes(mimeType)) {reject(`Unsupported conditioning image type: ${mimeType || 'none'}`,@@ -160,8 +170,15 @@ async function storeAsset({ callerId, mimeType, declaredSha256, body }) {// itself, so no separate ensureDir.const existing = await lstat(path).catch(() => null);if (existing?.isFile() && existing.size === body.length) {+ // A replay of staged bytes adds nothing, so it is exempt from the quota.await utimes(path, new Date(), new Date());} else {+ // Expired files are removed first so they never count against the caller.+ await sweepFederatedMediaAssets({ jobs, force: true }).catch((error) => {+ maintenance.markCurrentUnsettled();+ console.error(`❌ Federated media inbox sweep failed: ${error.message}`);+ });+ await enforceAssetQuota(callerId, body.length);await atomicWrite(path, body).catch((error) => {// The shared writer may have left an unknown temporary sibling, or// suppressed a cleanup error. Keep this admission unsettled rather than@@ -180,6 +197,35 @@ async function storeAsset({ callerId, mimeType, declaredSha256, body }) {};}+async function enforceAssetQuota(callerId, incomingBytes) {+ const ownerPrefix = `${federatedMediaAssetOwner(callerId)}-`;+ const entries = await readdir(PATHS.federatedMediaInbox).catch(() => []);+ const infos = await Promise.all(entries.map(async (name) => ({+ name,+ info: await lstat(join(PATHS.federatedMediaInbox, name)).catch(() => null),+ })));+ let totalBytes = 0;+ let callerCount = 0;+ let callerBytes = 0;+ for (const { name, info } of infos) {+ if (!info?.isFile()) continue;+ totalBytes += info.size;+ if (name.startsWith(ownerPrefix)) {+ callerCount += 1;+ callerBytes += info.size;+ }+ }+ const over = callerCount + 1 > FEDERATED_MEDIA_ASSET_CALLER_MAX_COUNT+ || callerBytes + incomingBytes > FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES+ || totalBytes + incomingBytes > FEDERATED_MEDIA_ASSET_TOTAL_MAX_BYTES;+ if (over) {+ reject('Conditioning image storage quota exceeded; wait for staged inputs to expire', 'MEDIA_PROVIDER_ASSET_QUOTA', 429, {+ maxCount: FEDERATED_MEDIA_ASSET_CALLER_MAX_COUNT,+ maxBytes: FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES,+ });+ }+}+/*** Metadata for one staged asset, so a consumer can skip a re-upload after a* restart. 404 covers absent, expired, and another caller's — seediff --git a/server/services/federatedMedia/assetStore.test.js b/server/services/federatedMedia/assetStore.test.jsindex da11fb3ebd..5d465742e4 100644--- a/server/services/federatedMedia/assetStore.test.js+++ b/server/services/federatedMedia/assetStore.test.js@@ -12,15 +12,19 @@ vi.mock('../../lib/fileUtils.js', async (load) => {} };});import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';-import { mkdtempSync, rmSync, writeFileSync, existsSync, utimesSync } from 'node:fs';+import { mkdtempSync, rmSync, writeFileSync, existsSync, utimesSync, truncateSync } from 'node:fs';import { join } from 'node:path';import { tmpdir } from 'node:os';import { PATHS } from '../../lib/fileUtils.js';import {FEDERATED_MEDIA_ASSET_TTL_MS,federatedMediaAssetId,+ federatedMediaAssetOwner,} from '../../lib/federatedMediaWire.js';import {+ FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES,+ FEDERATED_MEDIA_ASSET_CALLER_MAX_COUNT,+ FEDERATED_MEDIA_ASSET_TOTAL_MAX_BYTES,__resetFederatedAssetSweepForTests,collectActiveFederatedAssetBasenames,describeFederatedMediaAsset,@@ -66,7 +70,53 @@ const upload = async (callerId, body = PNG, mimeType = 'image/png', digest) => sbody,});+const distinctPng = (n) => Buffer.concat([PNG, Buffer.from(`-variant-${n}`)]);++// Sparse file: reports its logical size to lstat without writing the bytes.+const plantSparse = (callerId, tag, sizeBytes, ageMs = 0) => {+ const path = join(tempRoot, `${federatedMediaAssetOwner(callerId)}-${tag.padStart(64, '0')}.png`);+ writeFileSync(path, '');+ truncateSync(path, sizeBytes);+ const when = (Date.now() - ageMs) / 1000;+ utimesSync(path, when, when);+ return path;+};+describe('federated media asset store', () => {+ describe('storage quota', () => {+ it('refuses a caller beyond the per-caller count with 429, but still lets another caller upload and a replay refresh', async () => {+ for (let i = 0; i < FEDERATED_MEDIA_ASSET_CALLER_MAX_COUNT; i += 1) await upload('peer-a', distinctPng(i));+ await expect(upload('peer-a', distinctPng(9999))).rejects.toMatchObject({ status: 429, code: 'MEDIA_PROVIDER_ASSET_QUOTA' });+ await expect(upload('peer-a', distinctPng(0))).resolves.toMatchObject({ sizeBytes: distinctPng(0).length });+ await expect(upload('peer-b', distinctPng(9999))).resolves.toMatchObject({ assetId: expect.any(String) });+ });++ it('refuses a caller beyond the per-caller byte cap', async () => {+ plantSparse('peer-a', 'a', FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES - 10);+ await expect(upload('peer-a')).rejects.toMatchObject({ status: 429, code: 'MEDIA_PROVIDER_ASSET_QUOTA' });+ });++ it('refuses any caller once the store total is full', async () => {+ const chunk = FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES - 1;+ const chunks = Math.ceil(FEDERATED_MEDIA_ASSET_TOTAL_MAX_BYTES / chunk);+ for (let i = 0; i < chunks; i += 1) plantSparse(`other-${i}`, String(i), Math.min(chunk, FEDERATED_MEDIA_ASSET_TOTAL_MAX_BYTES - i * chunk));+ await expect(upload('peer-z')).rejects.toMatchObject({ status: 429, code: 'MEDIA_PROVIDER_ASSET_QUOTA' });+ });++ it('sweeps expired assets on upload before evaluating the cap, but keeps ones a live job pins', async () => {+ const stale = plantSparse('peer-a', 'b', FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES - 10, FEDERATED_MEDIA_ASSET_TTL_MS + 60_000);+ await expect(upload('peer-a')).resolves.toMatchObject({ sizeBytes: PNG.length });+ expect(existsSync(stale)).toBe(false);++ const pinned = plantSparse('peer-c', 'c', FEDERATED_MEDIA_ASSET_CALLER_MAX_BYTES - 10, FEDERATED_MEDIA_ASSET_TTL_MS + 60_000);+ const jobs = [{ status: 'running', params: { initImagePath: pinned } }];+ await expect(storeFederatedMediaAsset({+ callerId: 'peer-c', mimeType: 'image/png', declaredSha256: await sha256(PNG), body: PNG, jobs,+ })).rejects.toMatchObject({ code: 'MEDIA_PROVIDER_ASSET_QUOTA' });+ expect(existsSync(pinned)).toBe(true);+ });+ });+it('keeps input admission unsettled when shared persistence may leave an unknown partial temp file', async () => {failures.write = true;await expect(upload('peer-a')).rejects.toMatchObject({ code: 'EIO' });