Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Universe Builder — collection hook.
*
* Subscribes to mediaJobEvents and, for each completed image job that
* carries `params.universeRun.collectionId`, files the rendered filename
* into that collection.
*
* Mounted once at server boot from server/index.js so it can listen for
* the lifetime of the process. Failures here are logged but never thrown
* — a bookkeeping miss must not crash the server or fail the user's
* render.
*
* Re-export coalescing: when the route calls `registerUniverseBuilderRun`
* with a job count, every `addItem` for that run is wrapped in
* `withReexportSuppressed` so the per-image emits don't each schedule a
* debounced re-export of the whole universe. Once the last job in the run
* reaches a terminal state (completed/failed/canceled), the hook emits a
* single `emitRecordUpdated('universe', …)` and the subscription debounces
* one re-export. Server restart mid-run drops the tracker entry — the
* hook then falls back to per-image emits (current/legacy behavior) until
* the run finishes naturally.
*/
import { mediaJobEvents } from './mediaJobQueue/index.js';
import { addItem, ERR_DUPLICATE } from './mediaCollections.js';
import { withReexportSuppressed, emitRecordUpdated } from './sharing/recordEvents.js';
import { appendEntryImageRef, getUniverse, ENTRY_REF_KIND } from './universeBuilder.js';
import { fillImageSidecarFields } from './imageGen/local.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
// runId → { universeId, pending, universePromise? }. Pure in-memory; lost on restart.
// `universePromise` memoizes the universe doc for the lifetime of the batch so
// 160-image runs don't re-read universe-builder.json per completion.
const activeRuns = new Map();
// Completion handlers launched by the sync event listener and not yet finished.
// EventEmitter never awaits listeners, so this set is the only handle on "the
// hook's work for the events emitted so far has settled" — the test seam below
// drains it instead of guessing with sleeps.
const inFlight = new Set();
export function registerUniverseBuilderRun({ runId, universeId, jobCount }) {
if (!runId || !universeId || !Number.isFinite(jobCount) || jobCount <= 0) return;
activeRuns.set(runId, { universeId, pending: jobCount });
}
// A batch refused part-way by the media queue (#8326) never enqueues
// `unadmitted` of the jobs it registered. Drop them from the count so the run's
// coalesced emit still fires once the admitted ones finish — or now, when they
// already have (or none were admitted).
export function shrinkUniverseBuilderRun(runId, unadmitted) {
const entry = activeRuns.get(runId);
if (!entry || !(unadmitted > 0)) return;
entry.pending -= unadmitted;
if (entry.pending > 0) return;
activeRuns.delete(runId);
emitRecordUpdated('universe', entry.universeId);
}
// Returns the run's universeId when this terminal closes the batch (pending → 0),
// otherwise null. Callers use the return value as the "fire the final emit" signal.
function noteTerminal(runId) {
const entry = activeRuns.get(runId);
if (!entry) return null;
entry.pending -= 1;
if (entry.pending > 0) return null;
activeRuns.delete(runId);
return entry.universeId;
}
// Resolves the entry's canonical name (canon entries carry `.name`; variations
// and sheets carry only the compiled label). Returns null when nothing can be
// resolved — the caller falls back to `tag.label`.
function resolveEntryName(universe, entryRef) {
if (!universe || !entryRef) return null;
if (entryRef.kind === ENTRY_REF_KIND.CANON && entryRef.kindKey && entryRef.id) {
const list = universe[entryRef.kindKey];
const hit = Array.isArray(list) ? list.find((e) => e?.id === entryRef.id) : null;
return typeof hit?.name === 'string' ? hit.name : null;
}
return null;
}
// Build the universe-context object to merge into the image sidecar. Mirrors
// the migration's contract so new + backfilled renders carry identical shapes.
function buildSidecarPatch({ tag, universe }) {
const entryRef = tag.entryRef;
const universeName = typeof universe?.name === 'string' ? universe.name : null;
const patch = {
universeId: tag.universeId,
...(universeName ? { universeName } : {}),
...(tag.runId ? { universeRunId: tag.runId } : {}),
...(tag.label ? { entryLabel: tag.label } : {}),
};
if (!entryRef) {
// Legacy variations without stable ids: still tag the universe + label
// so search can find renders by universe even without entity granularity.
if (tag.category) patch.entryCategory = tag.category;
return patch;
}
patch.entryKind = entryRef.kind;
patch.entryId = entryRef.id;
if (entryRef.kind === ENTRY_REF_KIND.VARIATION && entryRef.categoryKey) {
patch.entryCategory = entryRef.categoryKey;
} else if (entryRef.kind === ENTRY_REF_KIND.CANON && entryRef.kindKey) {
patch.entryCategory = entryRef.kindKey;
}
const resolvedName = resolveEntryName(universe, entryRef) || tag.label || null;
if (resolvedName) patch.entryName = resolvedName;
return patch;
}
let completedHandler = null;
let terminalHandler = null;
export function initUniverseBuilderCollectionHook() {
// Idempotent: a stray double-init (test reload, future refactor) would
// otherwise register two listeners and double-file every completed image.
if (completedHandler) return;
// EventEmitter does not await async listeners and does not catch their
// rejections — any throw here would surface as an unhandled promise
// rejection (process-killing on Node ≥15). Use a sync listener that
// launches the async work with a top-level catch so this bookkeeping
// miss can never crash the server or fail the user's render.
//
// The work is admitted synchronously, inside the queue's completion fan-out,
// so a backup cut that is already draining waits for the collection filing,
// entry-ref append and sidecar enrichment instead of capturing part of them.
completedHandler = (job) => {
const handled = withBackupAssetPublication(async () => {
if (!job || job.kind !== 'image') return;
const tag = job.params?.universeRun;
if (!tag) return;
const hasCollection = typeof tag.collectionId === 'string' && tag.collectionId.length > 0;
const hasEntryAppend = !!(tag.entryRef && tag.universeId);
// Section-local canon renders (#1395) carry an `entryRef` + `universeId`
// but no `collectionId` — they durably append the render to the entry's
// `imageRefs[]` without filing into a gallery collection. Batch + base-
// style-probe renders carry a `collectionId`. Bail only when neither
// piece of work is requested.
if (!hasCollection && !hasEntryAppend) return;
const filename = job.result?.filename;
if (!filename || typeof filename !== 'string') return;
const runActive = tag.runId ? activeRuns.has(tag.runId) : false;
// Collection filing only when a collection target is present — section-
// local renders skip it (no gallery membership, just the entry-ref
// append below).
const addItemCall = () => {
if (!hasCollection) return Promise.resolve(null);
return addItem(tag.collectionId, { kind: 'image', ref: filename })
.then(() => 'added')
.catch((err) => {
// A duplicate (same filename rendered twice in the same run) is
// expected when batchPerVariation > 1 and the gen output collides.
if (err?.code === ERR_DUPLICATE) return 'duplicate';
console.log(`⚠️ universe-builder collection hook failed for ${filename}: ${err?.message || String(err)}`);
return 'failed';
});
};
// Append the rendered filename to the source entry's `imageRefs[]` so
// the Universe Builder can show the latest render as an avatar. Wrapped
// alongside `addItem` under the same re-export-suppression scope, so a
// single coalesced emit fires when the run finishes. Best-effort —
// bookkeeping must never crash the server or fail the user's render.
const appendEntryRefCall = () => {
if (!tag.entryRef || !tag.universeId) return Promise.resolve(null);
return appendEntryImageRef(tag.universeId, tag.entryRef, filename)
.then((res) => (res ? 'appended' : null))
.catch((err) => {
console.log(`⚠️ universe-builder entry-ref append failed for ${filename}: ${err?.message || String(err)}`);
return null;
});
};
// Enrich the image's sidecar with universe + entity context so
// MediaHistory can search for renders by character/place name and the
// lightbox can show which entity produced the image. The universe doc
// read is memoized per-run on activeRuns so 160-image batches read the
// ~700KB universe JSON once, not 160 times. Untracked runs (server
// restart mid-batch) fall back to a per-completion read — rare and
// bounded by the remaining job count.
const enrichSidecarCall = () => {
if (!tag.universeId) return Promise.resolve(null);
const entry = tag.runId ? activeRuns.get(tag.runId) : null;
if (entry && !entry.universePromise) {
entry.universePromise = getUniverse(tag.universeId).catch(() => null);
}
const universePromise = entry?.universePromise || getUniverse(tag.universeId).catch(() => null);
return universePromise
// Fills only absent keys, so a re-render of the same filename never
// retags it with a different universe's data.
.then((universe) => fillImageSidecarFields(filename, buildSidecarPatch({ tag, universe })))
.catch((err) => {
console.log(`⚠️ universe-builder sidecar enrich failed for ${filename}: ${err?.message || String(err)}`);
return null;
});
};
try {
// When the run is tracked, swallow the per-item emitRecordUpdated so a
// 160-image batch doesn't fan into 160 debounced re-exports. The final
// emit below fires once when pending hits zero.
// The three writes touch independent files (media-collections.json,
// universe-builder.json, the per-image sidecar) and have no ordering
// dependency — fire them in parallel so a large batch doesn't pay
// the per-completion serialization cost on every job.
const work = async () => {
const [filed, appended] = await Promise.all([addItemCall(), appendEntryRefCall(), enrichSidecarCall()]);
return { filed, appended };
};
const { filed, appended } = runActive && tag.universeId
? await withReexportSuppressed('universe', tag.universeId, work)
: await work();
if (filed === 'added') {
console.log(`🌍 universe-builder run=${tag.runId?.slice(0, 8)} category=${tag.category} → ${filename}`);
} else if (filed === 'duplicate') {
console.log(`🌍 universe-builder run=${tag.runId?.slice(0, 8)} category=${tag.category} duplicate skipped: ${filename}`);
} else if (!hasCollection && appended === 'appended') {
// Section-local render: no collection filing, just the durable
// imageRefs[] append onto the canon entry.
console.log(`🌍 universe section-local render → entry ${tag.entryRef?.id?.slice(0, 8)} ← ${filename}`);
}
} finally {
// `finally` (not the happy path) so an unexpected throw still
// decrements pending — otherwise one anomaly stalls the run's
// coalesced emit forever.
if (tag.runId) {
const completedUniverseId = noteTerminal(tag.runId);
if (completedUniverseId) emitRecordUpdated('universe', completedUniverseId);
}
}
}).catch((err) => {
// Last-resort net for synchronous throws (unexpected job shape, etc).
console.log(`⚠️ universe-builder collection hook crashed: ${err?.message || err}`);
});
inFlight.add(handled);
handled.finally(() => inFlight.delete(handled));
};
// Failed/canceled jobs still count toward run completion — otherwise a
// single failure mid-batch would leave `pending > 0` forever and the
// final emit would never fire.
terminalHandler = (job) => {
if (!job || job.kind !== 'image') return;
const tag = job.params?.universeRun;
if (!tag?.runId) return;
const completedUniverseId = noteTerminal(tag.runId);
if (completedUniverseId) emitRecordUpdated('universe', completedUniverseId);
};
mediaJobEvents.on('completed', completedHandler);
mediaJobEvents.on('failed', terminalHandler);
mediaJobEvents.on('canceled', terminalHandler);
console.log('🌍 Universe Builder collection hook initialized');
}
// Test-only reset hook so suites that re-init can do so cleanly. Removes the
// previously registered listeners so re-init doesn't leak handlers.
export const __testing = {
reset: () => {
if (completedHandler) {
mediaJobEvents.off('completed', completedHandler);
completedHandler = null;
}
if (terminalHandler) {
mediaJobEvents.off('failed', terminalHandler);
mediaJobEvents.off('canceled', terminalHandler);
terminalHandler = null;
}
activeRuns.clear();
},
getActiveRuns: () => new Map(activeRuns),
// Resolves once every completion handler launched so far (and any launched
// while draining) has finished. Rejects after `timeoutMs` naming how many are
// still pending, so a stalled append fails loudly instead of the caller
// polling a persisted-state predicate until its own budget runs out.
settled: async ({ timeoutMs = 10000 } = {}) => {
let timer;
const deadline = new Promise((_, reject) => {
timer = setTimeout(() => reject(new Error(`universe-builder hook: ${inFlight.size} completion handler(s) still pending after ${timeoutMs}ms`)), timeoutMs);
});
const drain = async () => {
while (inFlight.size > 0) await Promise.allSettled([...inFlight]);
};
await Promise.race([drain(), deadline]).finally(() => clearTimeout(timer));
},
};