Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { assertCatalogIdentityWritable } from './catalogDB/identityAccess.js';
// Direct file-upload + voice-memo attachment for catalog ingredients.
//
// The existing `catalogIngestSources.ingestFromVoice` runs audio through the
// scrap → LLM-extraction → review flow (it MINTS new ingredients). This module
// is the other seam: take a file the user dropped/picked, or a memo they just
// recorded, and attach it DIRECTLY to an ingredient that already exists.
//
// Bytes land in a peer-federating library dir (images/audio/videos) so the
// `media_key` reference federates through catalogSync while the file itself
// federates through peerMediaLibrarySync — the same two-layer path gallery
// attachments already use. Nothing here calls hashImageForManifest; the library
// replica hashes the file when it federates.
import { randomUUID } from 'crypto';
import { readFile } from 'fs/promises';
import { join } from 'path';
import { ensureDir, PATHS, atomicWrite, resolveImageInputPath } from '../lib/fileUtils.js';
import { extractPngGenerationMetadata, normalizeGenerationMetadata } from '../lib/pngMetadata.js';
import { ServerError } from '../lib/errorHandler.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { getIngredient, attachMedia } from './catalogDB.js';
import { readImageSidecar, saveUploadedGalleryImage } from './imageGen/local.js';
import { transcribe } from './voice/stt.js';
// media_key caption cap mirrors catalogMediaAttachSchema.caption (2 000 chars).
const MAX_CAPTION = 2_000;
// MIME → file extension for the non-image library dirs (images always normalize
// to .png in saveUploadedGalleryImage). Unknown-but-recognized-category MIMEs
// fall back to a sane container extension.
const AUDIO_EXT = {
'audio/webm': 'webm', 'audio/ogg': 'ogg', 'audio/wav': 'wav', 'audio/x-wav': 'wav',
'audio/wave': 'wav', 'audio/mpeg': 'mp3', 'audio/mp3': 'mp3', 'audio/mp4': 'm4a', 'audio/aac': 'm4a',
};
const VIDEO_EXT = {
'video/mp4': 'mp4', 'video/webm': 'webm', 'video/quicktime': 'mov', 'video/ogg': 'ogv',
};
const hasMetadata = (value) => value && typeof value === 'object' && !Array.isArray(value) && Object.keys(value).length > 0;
const mediaAttachOptions = ({ role = null, caption = null, metadata = {} } = {}) => {
const options = { role, caption };
const normalized = normalizeGenerationMetadata(metadata);
if (hasMetadata(normalized)) options.metadata = normalized;
return options;
};
/**
* Classify a browser-reported MIME into the media library it belongs in. Pure —
* unit-tested. Strips any `;codecs=…` parameter first. Returns
* `{ category, kind, ext }` or `null` for a MIME we don't federate (documents
* have no library dir yet, so attaching one would break references on peers).
*/
export function classifyUploadMime(mimeType) {
const mime = (mimeType || '').toLowerCase().split(';')[0].trim();
if (mime.startsWith('image/')) return { category: 'image', kind: 'reference', ext: 'png' };
if (mime.startsWith('audio/')) return { category: 'audio', kind: 'audio', ext: AUDIO_EXT[mime] || 'webm' };
if (mime.startsWith('video/')) return { category: 'video', kind: 'video', ext: VIDEO_EXT[mime] || 'mp4' };
return null;
}
/**
* Read the generation provenance for an existing gallery image. Sidecars are
* authoritative for normalized/generated assets; the PNG fallback covers a
* gallery key that arrived before its sidecar or was created by an older build.
* Metadata is best-effort: a corrupt sidecar must not make an otherwise valid
* image impossible to attach.
*/
export async function readImageGenerationMetadata(
mediaKey,
{
readSidecarFn = readImageSidecar,
readFileFn = readFile,
resolveImagePathFn = resolveImageInputPath,
} = {},
) {
const sidecar = await Promise.resolve(readSidecarFn(mediaKey)).catch(() => null);
const fromSidecar = normalizeGenerationMetadata(sidecar?.metadata);
if (hasMetadata(fromSidecar)) return fromSidecar;
const imagePath = resolveImagePathFn(mediaKey);
if (!imagePath) return {};
const bytes = await Promise.resolve(readFileFn(imagePath)).catch(() => null);
return bytes ? extractPngGenerationMetadata(bytes) : {};
}
// Write raw bytes into a federating library dir under a collision-free
// `upload-<uuid8>.<ext>` key. Returns the bare filename (the media_key).
async function persistLibraryFile(buffer, dir, ext) {
const filename = `upload-${randomUUID().slice(0, 8)}.${ext}`;
await ensureDir(dir);
await atomicWrite(join(dir, filename), buffer);
return filename;
}
// The media row first names bytes written just before it, so it commits under
// the backup lease (#9982): a cut copies files before it dumps rows, and a row
// that waits out the cut can only name a file the copy already took.
const attachUnderLease = (attachMediaFn, ...args) => withBackupAssetPublication(() => attachMediaFn(...args));
async function requireIngredient(getIngredientFn, ingredientId) {
const ingredient = await getIngredientFn(ingredientId);
if (!ingredient) throw new ServerError('Ingredient not found', { status: 404 });
return ingredient;
}
/**
* Persist an uploaded file into the media library and attach it to an
* ingredient. Images route through `saveUploadedGalleryImage` (magic-byte
* sniffed, re-encoded to PNG, EXIF-baked); audio/video are stored as-is. Returns
* the attached media row. `deps` lets tests run without DB/disk.
*/
export async function uploadIngredientMediaFile(
{ ingredientId, dataBase64, mimeType, filename = null, role = null, caption = null } = {},
{
getIngredientFn = getIngredient,
assertWritableFn = assertCatalogIdentityWritable,
attachMediaFn = attachMedia,
saveImageFn = saveUploadedGalleryImage,
persistFileFn = persistLibraryFile,
} = {},
) {
await assertWritableFn(ingredientId);
await requireIngredient(getIngredientFn, ingredientId);
const classified = classifyUploadMime(mimeType);
if (!classified) {
throw new ServerError(`Unsupported file type "${mimeType}" — attach an image, audio, or video file.`, {
status: 422, code: 'UNSUPPORTED_MEDIA',
});
}
const meta = { role, caption };
if (classified.category === 'image') {
// saveUploadedGalleryImage validates size + format and 400s on a non-image.
const sourceMetadata = extractPngGenerationMetadata(Buffer.from(dataBase64, 'base64'));
const saved = await saveImageFn(dataBase64);
const metadata = hasMetadata(saved?.metadata) ? saved.metadata : sourceMetadata;
const mediaKey = saved.filename;
console.log(`📎 Catalog media upload: image ${mediaKey} → ingredient ${ingredientId}`);
return attachUnderLease(attachMediaFn, ingredientId, mediaKey, classified.kind, mediaAttachOptions({ ...meta, metadata }));
}
const buffer = Buffer.from(dataBase64, 'base64');
if (buffer.length === 0) throw new ServerError('Empty file upload', { status: 400, code: 'VALIDATION_ERROR' });
const dir = classified.category === 'audio' ? PATHS.audio : PATHS.videos;
const mediaKey = await persistFileFn(buffer, dir, classified.ext);
console.log(`📎 Catalog media upload: ${classified.kind} ${mediaKey} → ingredient ${ingredientId} (${(buffer.length / 1024).toFixed(0)}KB, ${filename || 'unnamed'})`);
return attachUnderLease(attachMediaFn, ingredientId, mediaKey, classified.kind, mediaAttachOptions(meta));
}
/**
* Attach a recorded voice memo: transcribe the WAV via Whisper, persist the
* audio into data/audio, and attach a `kind:'audio'` media row with the
* transcript in `caption`. Persist happens AFTER transcription so a failed
* transcript doesn't orphan a file (mirrors ingestFromVoice). An empty
* transcript still attaches the audio (the recording is the primary artifact;
* the client already guards against a silent mic). Returns `{ media, transcript }`.
*/
export async function recordIngredientVoiceMemo(
{ ingredientId, audioBase64, mimeType = 'audio/wav', role = 'voice-memo' } = {},
{
getIngredientFn = getIngredient,
assertWritableFn = assertCatalogIdentityWritable,
attachMediaFn = attachMedia,
transcribeFn = transcribe,
persistFileFn = persistLibraryFile,
} = {},
) {
await assertWritableFn(ingredientId);
await requireIngredient(getIngredientFn, ingredientId);
const buffer = Buffer.from(audioBase64, 'base64');
if (buffer.length === 0) throw new ServerError('Voice memo audio was empty', { status: 400, code: 'VALIDATION_ERROR' });
const { text } = await transcribeFn(buffer, { mimeType });
const transcript = (text || '').trim().slice(0, MAX_CAPTION);
const ext = AUDIO_EXT[(mimeType || '').toLowerCase().split(';')[0].trim()] || 'wav';
const mediaKey = await persistFileFn(buffer, PATHS.audio, ext);
console.log(`🎙️ Catalog voice memo: ${mediaKey} → ingredient ${ingredientId} (${transcript.length} chars)`);
const media = await attachUnderLease(attachMediaFn, ingredientId, mediaKey, 'audio', mediaAttachOptions({ role, caption: transcript || null }));
return { media, transcript };
}