Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Durable consumer-side execution core for federated media jobs, shared by
* every kind (audio, image, video).
*
* The local media-job UUID is the provider Idempotency-Key. A worker restart
* therefore replays the same submission, recovers the provider job, and keeps
* polling instead of creating duplicate paid/GPU work. Provider URLs are never
* accepted from the wire: every request derives the fixed v1 endpoint from the
* locally configured peer.
*
* Everything kind-specific — which marker shape is persisted, how the wire
* request is built from it, where the verified bytes land, and what a finished
* render has to register locally (an image sidecar, a video-history row) — is
* supplied by the caller. The retry/cancel/idempotency/integrity semantics are
* NOT: they are the part that must stay identical across kinds, which is why
* this module owns them instead of each adapter re-deriving them.
*
* A kind whose request body cannot be built without talking to the peer first
* (image conditioning has to be uploaded and turned into asset ids) gets this
* run's own `requestJson` handed to `buildRequest` rather than a routine of its
* own here. The cross-kind concept is "an authenticated sub-request inside the
* run's envelope"; WHAT is sent stays with the kind that knows — see
* federatedMedia/inputAssets.js.
*/
import { maintenance } from '../../lib/maintenanceAdmission.js';
import { createWriteStream } from 'node:fs';
import { mkdir, rename, stat, unlink } from 'node:fs/promises';
import { createHash } from 'node:crypto';
import { join } from 'node:path';
import { Readable, Transform } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { sha256File } from '../../lib/fileUtils.js';
import {
FEDERATED_MEDIA_WIRE_VERSION,
federatedMediaProviderJobSchema,
} from '../../lib/federatedMediaWire.js';
import { peerFetch } from '../../lib/peerHttpClient.js';
import { peerBaseUrl } from '../../lib/peerUrl.js';
import { readResponseJson } from '../../lib/readResponseJson.js';
import { resolveFederatedMediaProvider } from '../federatedMediaConsumer.js';
import { getPeers } from '../instances.js';
import { isTailnetPeer } from '../../lib/tailnetPeer.js';
const RETRYABLE_HTTP_STATUSES = new Set([408, 425, 429]);
const PERMANENT_SELECTION_CODES = new Set([
'MEDIA_PROVIDER_PEER_DISABLED',
'MEDIA_PROVIDER_NOT_CONFIGURED',
'MEDIA_PROVIDER_SELECTION_INVALID',
'MEDIA_PROVIDER_MODEL_NOT_ALLOWED',
]);
const NON_RETRYABLE_FILE_CODES = new Set(['EACCES', 'ENOSPC', 'EPERM', 'EROFS']);
const RETRYABLE_TRANSPORT_CODES = new Set([
'ECONNREFUSED',
'ECONNRESET',
'EHOSTUNREACH',
'ENETUNREACH',
'ETIMEDOUT',
'UND_ERR_CONNECT_TIMEOUT',
'UND_ERR_SOCKET',
]);
export const remoteMediaError = (message, details = {}) => Object.assign(new Error(message), details);
const isRetryableStatus = (status) => RETRYABLE_HTTP_STATUSES.has(status) || status >= 500;
const isRetryableTransportError = (error) =>
error?.retryable === true
|| error?.name === 'AbortError'
|| error?.name === 'TimeoutError'
|| error?.name === 'TypeError'
|| RETRYABLE_TRANSPORT_CODES.has(error?.code);
/**
* Build a durable remote executor for one media kind.
*
* @param {object} config
* @param {'audio'|'image'|'video'} config.kind - Wire kind this executor speaks.
* @param {string} config.label - Human noun used in progress/failure messages.
* @param {import('events').EventEmitter} config.events - The kind's gen event emitter.
* @param {import('zod').ZodTypeAny} config.markerSchema - Schema for `params.remoteMedia`.
* @param {(marker: object, ctx: object) => object|Promise<object>} config.buildRequest -
* Wire submission body builder. `ctx` carries this run's `requestJson` — the
* same authenticated, in-envelope request helper the executor uses itself, so
* a body that has to make a sub-request while it is being built (staging
* conditioning images today) does so inside the run's retry, cancel and
* timeout envelope — plus `emitStatus` to narrate it. WHAT that sub-request is
* stays kind-specific; the executor only lends the channel.
* @param {(ctx: object) => {dir: string, filename: string}} config.resolveDestination -
* Where the verified result lands. Called per job so a PATHS proxy stays live.
* @param {(work: Function, ctx: object) => Promise<object>} [config.publishResult]
* Owns verified-file installation plus finalize, including replacement rollback.
* Downloads complete before admission; matching-file replays still finalize.
* @param {(ctx: object) => Promise<object>} config.finalize - Register the downloaded
* result locally; its return value is merged into the `completed` event payload.
* Its ctx carries `renderStartedAtMs` (this install's ingestion instant) for
* `renderTimingFields` — see the call site for what that span covers.
*/
export function createRemoteMediaExecutor({
kind,
label,
events,
markerSchema,
buildRequest,
resolveDestination,
finalize,
publishResult = work => work(),
}) {
let pollDelayMs = 1_000;
let retryDelayMs = 2_000;
let requestTimeoutMs = 30_000;
const activeJobs = new Map();
const canceledError = () => remoteMediaError(`Remote ${label} generation canceled`, { canceled: true });
function emitActivity(jobId) {
events.emit('activity', { generationId: jobId });
}
function emitStatus(jobId, message) {
emitActivity(jobId);
events.emit('status', { generationId: jobId, message });
}
async function waitForRetry(state, delayMs) {
emitActivity(state.jobId);
if (delayMs <= 0) return;
await new Promise((resolve) => {
let settled = false;
const finish = () => {
if (settled) return;
settled = true;
clearTimeout(timer);
if (state.wake === finish) state.wake = null;
resolve();
};
const timer = setTimeout(finish, delayMs);
timer.unref?.();
state.wake = finish;
});
}
// `timeoutScale` multiplies the JSON request timeout for a sub-request that
// legitimately outruns it (a multi-megabyte conditioning upload). It stays a
// scale rather than an absolute so callers outside this module never have to
// know — or fall out of sync with — the configured base timeout.
async function withRequest(
state,
fn,
{ respectCancel = true, timeoutScale = 1, timeoutMs = requestTimeoutMs * timeoutScale } = {},
) {
const controller = new AbortController();
const timer = timeoutMs === null ? null : setTimeout(() => controller.abort(), timeoutMs);
timer?.unref?.();
state.requestController = controller;
if (respectCancel && state.cancelRequested) controller.abort();
try {
return await fn(controller.signal);
} finally {
if (timer) clearTimeout(timer);
if (state.requestController === controller) state.requestController = null;
}
}
async function findPeer(peerId, { standingRoute = false } = {}) {
const peers = await getPeers();
const peer = peers.find((candidate) => candidate.id === peerId);
if (!peer || peer.enabled === false) {
throw remoteMediaError('Selected media provider peer is no longer available', {
code: 'MEDIA_PROVIDER_PEER_UNAVAILABLE',
});
}
// A standing route's tailnet gate is checked at enqueue, but the peer record
// it checked can change while the job sits queued or reconciles across a
// restart — a host edited from a .ts.net name to a LAN address would
// otherwise carry the visual prompt over an unauthenticated hop on the very
// next request. Every request path (submit, poll, cancel, result) resolves
// its peer here, so re-checking at this one chokepoint covers all of them.
// ADR docs/decisions/2026-08-20-federated-visual-prompts.md, rule 5.
if (standingRoute && !isTailnetPeer(peer)) {
throw remoteMediaError('Media provider peer is no longer a Tailscale host', {
code: 'MEDIA_ROUTING_PEER_NOT_TAILNET',
});
}
return peer;
}
async function requestJson(state, path, options = {}, requestOptions = {}) {
const peer = await findPeer(state.peerId, { standingRoute: state.standingRoute });
const { response, body } = await withRequest(
state,
async (signal) => {
const response = await peerFetch(`${peerBaseUrl(peer)}${path}`, {
redirect: 'error',
...options,
signal,
}, peer);
const body = await readResponseJson(response, { fallback: null, emptyValue: null });
return { response, body };
},
requestOptions,
);
if (!response.ok) {
const code = typeof body?.code === 'string' ? body.code : `HTTP_${response.status}`;
throw remoteMediaError(`Remote media provider rejected the request (${code})`, {
code,
status: response.status,
retryable: isRetryableStatus(response.status),
});
}
return body;
}
function parseProviderJob(body, expectedId) {
const parsed = federatedMediaProviderJobSchema.safeParse(body);
if (!parsed.success || parsed.data.kind !== kind || (expectedId && parsed.data.id !== expectedId)) {
throw remoteMediaError('Remote media provider returned an invalid wire-v1 job projection', {
code: 'MEDIA_PROVIDER_INVALID_JOB_RESPONSE',
});
}
return parsed.data;
}
async function preflight(state, selection) {
while (true) {
if (state.cancelRequested) throw canceledError();
const peer = await findPeer(state.peerId, { standingRoute: state.standingRoute });
try {
await withRequest(
state,
(signal) => resolveFederatedMediaProvider(peer, selection, { signal }),
);
return;
} catch (error) {
if (state.cancelRequested) throw canceledError();
if (PERMANENT_SELECTION_CODES.has(error?.code)) throw error;
emitStatus(state.jobId, 'Waiting for remote provider readiness');
await waitForRetry(state, retryDelayMs);
}
}
}
async function submitOrRecover(state, request) {
const path = '/api/federation/media/v1/jobs';
while (true) {
// Once an attempt begins, an abort is ambiguous: the provider may have
// accepted the body before the connection broke. Keep replaying with the
// same key (even after local cancellation) until its job id is recovered.
state.submissionMayExist = true;
try {
const body = await requestJson(state, path, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Idempotency-Key': state.jobId,
},
body: JSON.stringify(request),
}, { respectCancel: !state.cancelRequested });
return parseProviderJob(body);
} catch (error) {
if (!isRetryableTransportError(error)) throw error;
emitStatus(state.jobId, state.cancelRequested
? 'Recovering remote job before cancellation'
: 'Waiting to submit to the remote provider');
// Cancellation must still recover the possibly-accepted submission before
// it can target the provider job. Keep the normal backoff while doing so:
// a disconnected peer must not turn a durable cancel into a hot loop.
await waitForRetry(state, retryDelayMs);
}
}
}
async function sendRemoteCancel(state, remoteJobId) {
while (true) {
try {
const body = await requestJson(
state,
`/api/federation/media/v1/jobs/${remoteJobId}/cancel`,
{ method: 'POST' },
{ respectCancel: false },
);
state.cancelSent = true;
return parseProviderJob(body, remoteJobId);
} catch (error) {
// A terminal provider job returns 409. The local user still asked to
// cancel, so do not import a result that won the race.
if (error?.status === 409) throw canceledError();
if (!isRetryableTransportError(error)) throw error;
emitStatus(state.jobId, 'Waiting to cancel the remote job');
await waitForRetry(state, retryDelayMs);
}
}
}
function emitProviderProgress(state, job) {
emitActivity(state.jobId);
if (typeof job.progress === 'number') {
events.emit('progress', {
generationId: state.jobId,
progress: job.progress,
message: 'Rendering on remote provider',
...(typeof job.etaMs === 'number' ? { etaMs: job.etaMs } : {}),
});
return;
}
const position = typeof job.position === 'number' ? ` (position ${job.position})` : '';
emitStatus(state.jobId, job.status === 'queued'
? `Queued on remote provider${position}`
: 'Rendering on remote provider');
}
async function pollProviderJob(state, initial) {
let current = initial;
const rememberTerminal = job => {
if (!['completed', 'failed', 'canceled'].includes(job.status)) return;
state.remoteTerminal = true;
state.remoteTerminalJob = { id: job.id, status: job.status };
};
while (true) {
rememberTerminal(current);
if (state.cancelRequested && !state.cancelSent) {
current = await sendRemoteCancel(state, current.id);
rememberTerminal(current);
}
if (state.cancelRequested && ['completed', 'failed', 'canceled'].includes(current.status)) {
throw canceledError();
}
if (current.status === 'completed') {
if (!current.result) {
throw remoteMediaError('Remote provider completed without result metadata', {
code: 'MEDIA_PROVIDER_RESULT_MISSING',
});
}
return current;
}
if (current.status === 'failed') {
throw remoteMediaError(`Remote provider job failed (${current.failure?.code || 'MEDIA_PROVIDER_JOB_FAILED'})`, {
code: current.failure?.code || 'MEDIA_PROVIDER_JOB_FAILED',
});
}
if (current.status === 'canceled') {
throw remoteMediaError('Remote provider canceled the job', { code: 'MEDIA_PROVIDER_JOB_CANCELED' });
}
emitProviderProgress(state, current);
await waitForRetry(state, pollDelayMs);
try {
const body = await requestJson(
state,
`/api/federation/media/v1/jobs/${current.id}`,
{},
{ respectCancel: true },
);
current = parseProviderJob(body, current.id);
} catch (error) {
if (state.cancelRequested && error?.name === 'AbortError') continue;
if (!isRetryableTransportError(error)) throw error;
emitStatus(state.jobId, 'Remote provider temporarily unavailable');
await waitForRetry(state, retryDelayMs);
}
}
}
async function existingResultMatches(path, metadata) {
const info = await stat(path).catch(() => null);
if (!info?.isFile() || info.size !== metadata.sizeBytes) return false;
const digest = await sha256File(path).catch(() => null);
return digest === metadata.sha256;
}
function responseStream(body) {
if (!body) throw remoteMediaError('Remote provider returned an empty result body');
return typeof body.getReader === 'function' ? Readable.fromWeb(body) : Readable.from(body);
}
async function downloadOnce(state, remoteJob) {
const metadata = remoteJob.result;
const { dir, filename } = resolveDestination({ jobId: state.jobId, remoteJob });
const finalPath = join(dir, filename);
const partialPath = join(dir, `.${filename}.partial`);
await mkdir(dir, { recursive: true });
if (await existingResultMatches(finalPath, metadata)) return { filename, dir, path: finalPath };
await unlink(partialPath).catch((error) => { if (error.code !== 'ENOENT') maintenance.markCurrentUnsettled(); });
const peer = await findPeer(state.peerId, { standingRoute: state.standingRoute });
// Keep the controller live for the body stream so cancel()/the queue
// watchdog can interrupt a stalled transfer. There is intentionally no
// fixed wall-clock timeout: large renders over a slow Tailnet are valid
// work, and chunk activity keeps the queue's idle watchdog fresh.
return withRequest(state, async (signal) => {
const response = await peerFetch(
`${peerBaseUrl(peer)}/api/federation/media/v1/jobs/${remoteJob.id}/result`,
{ redirect: 'error', signal },
peer,
);
if (!response.ok) {
throw remoteMediaError(`Remote result download failed (HTTP_${response.status})`, {
status: response.status,
retryable: isRetryableStatus(response.status),
});
}
const contentLength = Number(response.headers.get('content-length'));
const contentType = response.headers.get('content-type')?.split(';', 1)[0]?.trim();
const advertisedHash = response.headers.get('x-content-sha256')?.toLowerCase();
if (!Number.isInteger(contentLength) || contentLength !== metadata.sizeBytes
|| contentType !== metadata.mimeType || advertisedHash !== metadata.sha256) {
throw remoteMediaError('Remote result headers did not match the validated metadata', {
code: 'MEDIA_PROVIDER_RESULT_HEADERS_INVALID',
});
}
let sizeBytes = 0;
let lastActivityAt = 0;
const hasher = createHash('sha256');
const meter = new Transform({
transform(chunk, _encoding, callback) {
sizeBytes += chunk.length;
if (sizeBytes > metadata.sizeBytes) {
callback(remoteMediaError('Remote result exceeded its advertised size'));
return;
}
hasher.update(chunk);
const now = Date.now();
if (now - lastActivityAt >= 1_000) {
lastActivityAt = now;
emitActivity(state.jobId);
}
callback(null, chunk);
},
});
let verified = false;
try {
await pipeline(responseStream(response.body), meter, createWriteStream(partialPath, { flags: 'wx' }));
const digest = hasher.digest('hex');
if (sizeBytes !== metadata.sizeBytes || digest !== metadata.sha256) {
throw remoteMediaError('Remote result failed byte-integrity verification', {
code: 'MEDIA_PROVIDER_RESULT_INTEGRITY_FAILED',
});
}
// Promotion waits for the adapter's publication boundary. The verified
// staging file is never named by a history record.
verified = true;
return { filename, dir, path: finalPath, stagedPath: partialPath };
} finally {
if (!verified) await unlink(partialPath).catch((error) => { if (error.code !== 'ENOENT') maintenance.markCurrentUnsettled(); });
}
}, { timeoutMs: null });
}
async function downloadResult(state, remoteJob) {
while (true) {
if (state.cancelRequested) throw canceledError();
emitStatus(state.jobId, `Downloading verified remote ${label}`);
try {
return await downloadOnce(state, remoteJob);
} catch (error) {
if (state.cancelRequested) throw canceledError();
const retryable = !NON_RETRYABLE_FILE_CODES.has(error?.code)
&& !error?.code?.startsWith('MEDIA_PROVIDER_RESULT_')
&& isRetryableTransportError(error);
if (!retryable) throw error;
emitStatus(state.jobId, `Remote ${label} transfer interrupted; retrying`);
await waitForRetry(state, retryDelayMs);
}
}
}
async function runRemote(state, marker) {
// Awaited, and handed this run's own request channel: an image/video marker
// resolves its persisted LOCAL conditioning paths into provider asset ids
// here, inside this run's retry, cancel and timeout envelope rather than
// before it. See federatedMedia/inputAssets.js for what that entails.
const request = await buildRequest(marker, {
requestJson: (path, options, requestOptions) => requestJson(state, path, options, requestOptions),
emitStatus: (message) => emitStatus(state.jobId, message),
});
const selection = { kind, engine: request.engine, modelId: request.modelId };
if (!marker.reconcile) await preflight(state, selection);
if (state.cancelRequested && !marker.reconcile && !state.submissionMayExist) throw canceledError();
const submitted = await submitOrRecover(state, request);
const completed = await pollProviderJob(state, submitted);
const downloaded = await downloadResult(state, completed);
state.finalizing = true;
let local;
try {
local = await publishResult(async () => {
// Atomic installation preserves an existing path until its replacement lands.
if (downloaded.stagedPath) await rename(downloaded.stagedPath, downloaded.path);
return finalize({
jobId: state.jobId,
peerId: state.peerId,
request,
remoteJob: completed,
// Local-only evidence for the consumer sidecar; never part of the wire request.
inputAssets: marker.inputAssets,
// Wall-clock render timing (#5878). Measured from THIS install's
// ingestion of the job — submission, the peer's own queue wait and
// render, download, verification — because that whole span is what the
// user waited through here. The peer's internal render time is not on
// the wire, and asking for it would be a status payload crossing the
// federation boundary.
renderStartedAtMs: state.renderStartedAtMs,
...downloaded,
});
}, { ...downloaded, jobId: state.jobId });
} finally {
if (downloaded.stagedPath) await unlink(downloaded.stagedPath).catch(error => {
if (error.code !== 'ENOENT') maintenance.markCurrentUnsettled();
});
}
state.finalizing = false;
return {
...local,
federatedMedia: {
wireVersion: FEDERATED_MEDIA_WIRE_VERSION,
peerId: state.peerId,
remoteJobId: completed.id,
},
};
}
/** Queue entry point — mirrors the local generator's `generateX(params)` shape. */
async function run(params) {
const marker = markerSchema.safeParse(params?.remoteMedia);
if (!marker.success) {
if (params?.remoteMedia?.reconcile === true) maintenance.markCurrentUnsettled();
events.emit('failed', {
generationId: params?.jobId,
error: `Remote ${label} job has invalid persisted routing metadata`,
});
return;
}
// The queue serializes peer work; also refuse overlapping attempts for
// the same idempotency key at the executor boundary.
if (activeJobs.has(params.jobId)) {
throw remoteMediaError('This remote job is already being observed', { code: 'MEDIA_PROVIDER_RECONCILIATION_ACTIVE' });
}
const state = {
jobId: params.jobId,
peerId: marker.data.peerId,
cancelRequested: marker.data.cancelRequested === true,
cancelSent: false,
submissionMayExist: marker.data.reconcile === true,
// Absent on an interactive marker (and on any marker queued before this
// shipped), which reads correctly as "not a standing route".
standingRoute: marker.data.standingRoute === true,
requestController: null,
wake: null,
// The instant the media-job queue handed this job to us; `finalize` turns
// it into the record's render-timing fields (#5878).
//
// NOT stamped on a reconcile. That path re-enters `run()` after a restart
// for a job that was already submitted — `Date.now()` here would measure
// only the post-restart download-and-verify tail, so a 20-minute render
// interrupted by a `pm2 restart` would land claiming it took 8 seconds.
// The pre-restart instant is not recoverable (it lived in this in-memory
// state, not on the persisted marker), so report the honest unknown:
// `renderTimingFields(null)` yields `{}` and the card shows no duration.
renderStartedAtMs: marker.data.reconcile === true ? null : Date.now(),
};
activeJobs.set(params.jobId, state);
let event;
let payload;
try {
const result = await runRemote(state, marker.data);
// This is a transient executor verdict, never a provider/caller field.
// The queue may release owned inputs only AFTER its terminal row is durable.
event = 'completed';
payload = { generationId: params.jobId, ...result,
remoteInputsDisposable: !state.finalizing && (state.remoteTerminal === true || !state.submissionMayExist) };
} catch (error) {
// An unreachable/revoked peer is not proof that its admitted render
// stopped. Keep local ownership until completion can be reconciled.
if ((state.submissionMayExist && !state.remoteTerminal) || state.finalizing) maintenance.markCurrentUnsettled();
event = 'failed';
payload = {
generationId: params.jobId,
remoteInputsDisposable: !state.finalizing && (state.remoteTerminal === true || !state.submissionMayExist),
error: error?.canceled
? `Remote ${label} generation canceled`
: (error?.message || `Remote ${label} generation failed`),
};
} finally {
activeJobs.delete(params.jobId);
}
// Created locally only after executor teardown, never accepted from the
// provider response. Safe disposal can also mean "never submitted"; that
// does not certify termination of a previously admitted peer job.
if (marker.data.reconcile === true && state.remoteTerminalJob && !state.finalizing) {
payload.remoteRecoverySettlement = {
jobId: state.jobId, peerId: state.peerId, remoteJobId: state.remoteTerminalJob.id,
status: state.remoteTerminalJob.status, executorSettled: true,
};
}
events.emit(event, payload);
}
function cancel(jobId) {
const state = activeJobs.get(jobId);
if (!state) return;
state.cancelRequested = true;
state.requestController?.abort();
state.wake?.();
}
function configureForTests(options = {}) {
pollDelayMs = options.pollDelayMs ?? 0;
retryDelayMs = options.retryDelayMs ?? 0;
requestTimeoutMs = options.requestTimeoutMs ?? 1_000;
}
function resetForTests() {
for (const state of activeJobs.values()) {
state.cancelRequested = true;
state.requestController?.abort();
state.wake?.();
}
activeJobs.clear();
pollDelayMs = 1_000;
retryDelayMs = 2_000;
requestTimeoutMs = 30_000;
}
return { run, cancel, configureForTests, resetForTests };
}