Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Media asset index — public entry / boot wiring (#1000).
*
* The index is a DERIVED Postgres mirror of on-disk media (see db.js). This
* module owns its lifecycle:
* - initMediaAssetIndex() — called once at boot (after the DB gate). Subscribes
* to the existing image/video `completed` events so a freshly generated
* asset is indexed immediately, then runs a full reconcile so the index
* matches disk regardless of any events missed while the server was down.
* - unindexImage() / unindexVideo() — the delete-side mirror of those hooks,
* called by the media delete paths so a row dies with its file (#2738).
* - The reconcile + row I/O live in db.js; the pure transforms in logic.js.
*
* No hot-path generator edits: we hang off the `imageGenEvents` /
* `videoGenEvents` 'completed' emitters that already fire on every render. The
* handlers read the just-written sidecar / history entry and upsert one row.
*
* Escape hatch: under MEMORY_BACKEND=file or a test runner there's no Postgres,
* so the index is simply not maintained — the gallery/history still serve from
* disk. init() no-ops in that case (mirrors how catalog features disable).
*/
import { isTestRunner } from '../../lib/runtimeEnv.js';
import { checkHealth, ensureSchema } from '../../lib/db.js';
import { imageGenEvents } from '../imageGenEvents.js';
import { videoGenEvents } from '../videoGen/events.js';
import { upsertAsset, removeAsset, reconcileMediaAssets, queueMediaIndexRefresh } from './db.js';
import { imageMediaKey, imageToRow, videoMediaKey, videoToRow } from './logic.js';
export { reconcileMediaAssets } from './db.js';
let subscribed = false;
function isEscapeHatch() {
return process.env.MEMORY_BACKEND === 'file' || isTestRunner();
}
// Index a single just-generated image. The 'completed' event carries the
// filename; the full metadata is in the sidecar the generator just wrote.
export async function indexImage({ filename, temp } = {}) {
if (isEscapeHatch()) return;
if (typeof filename !== 'string' || !filename) return;
// A non-gallery temp render (issue #2264, Image Cleaner GPU pass) has no
// gallery file or sidecar — skip indexing it. It lives in imageCleanTmp and
// is consumed by an explicit result-fetch, never listed in the gallery.
if (temp) return;
return queueMediaIndexRefresh(async () => {
const { readImageSidecar } = await import('../imageGen/local.js');
const { metadata } = await readImageSidecar(filename);
// Match disk gallery's timestamp fallback for uploads without a sidecar.
let createdAt = metadata.createdAt;
if (!createdAt) {
const [{ stat }, { join }, { PATHS }] = await Promise.all([
import('node:fs/promises'), import('node:path'), import('../../lib/fileUtils.js'),
]);
createdAt = (await stat(join(PATHS.images, filename))).birthtime.toISOString();
}
const row = imageToRow({ filename, path: `/data/images/${filename}`, createdAt, ...metadata });
await upsertAsset(row).catch((err) => console.error(`❌ Media index image upsert failed: ${err.message}`));
});
}
// Index the just-generated video(s). The 'completed' event carries the job id
// (generationId); the full metadata is the matching video-history entry. A
// multi-format HTML composition (#8960) registers one entry per format and
// lists their ids in `videos`, none of which is the job id itself.
async function onVideoCompleted({ generationId, videos } = {}) {
const ids = Array.isArray(videos) ? videos.map((video) => video?.id) : [generationId];
const wanted = new Set(ids.filter((id) => typeof id === 'string' && id));
if (!wanted.size) return;
return queueMediaIndexRefresh(async () => {
const { loadHistory } = await import('../videoGen/local.js');
const history = await loadHistory().catch(() => []);
const entries = Array.isArray(history) ? history.filter((h) => wanted.has(h.id)) : [];
for (const entry of entries) {
await upsertAsset(videoToRow(entry)).catch((err) => console.error(`❌ Media index video upsert failed: ${err.message}`));
}
});
}
// Post-delete hooks tolerate index failure until boot reconciliation. A caller
// retaining its history row and bytes can use strict removal to fail closed.
// Test/file backends have no derived Postgres index to maintain.
async function unindexKey(mediaKey, kind, { strict = false } = {}) {
if (!mediaKey || isEscapeHatch()) return;
const removal = queueMediaIndexRefresh(() => removeAsset(mediaKey));
if (strict) return removal;
await removal.catch((err) => console.error(`❌ Media index ${kind} remove failed: ${err.message}`));
}
/**
* Drop a deleted image's row. `filename` is the gallery filename — the same ref
* onImageCompleted indexed it under, so the key matches the row that was written.
* Never throws.
*/
export async function unindexImage(filename) {
await unindexKey(imageMediaKey(filename), 'image');
}
/**
* Drop a video's row. `id` is the job id (NOT the filename).
* `strict` propagates failure when a caller still holds the history entry and
* bytes for a retry; ordinary post-delete hooks remain non-fatal.
*/
export async function unindexVideo(id, options = {}) {
await unindexKey(videoMediaKey(id), 'video', options);
}
/**
* Boot init: subscribe the completed-hooks (once) + run a full reconcile.
* No-op under the escape hatch or when Postgres is unreachable (the boot DB
* gate already fail-fasts a required-but-missing DB; this just stays quiet).
*/
export async function initMediaAssetIndex() {
if (isEscapeHatch()) {
console.log('🗂️ Media asset index disabled (escape hatch / test) — serving from disk only');
return { ok: false, reason: 'escape-hatch' };
}
const health = await checkHealth();
if (!health.connected) return { ok: false, reason: 'db-unreachable' };
// Self-sufficient like the CD backend: ensureSchema is idempotent + in-flight
// deduped, so calling it here is safe regardless of boot ordering.
await ensureSchema();
if (!subscribed) {
// The completed handlers run outside the request lifecycle (event emitter),
// so an uncaught throw would crash Node — each handler is self-contained and
// its upsert .catch()es. Wrap the dispatch too, defensively.
imageGenEvents.on('completed', (p) => { indexImage(p).catch((err) => console.error(`❌ Media index image hook: ${err.message}`)); });
videoGenEvents.on('completed', (p) => { onVideoCompleted(p).catch((err) => console.error(`❌ Media index video hook: ${err.message}`)); });
subscribed = true;
}
return reconcileMediaAssets();
}