Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

14.8 KB
Raw Blame
/** Upload staging, validation, persistence, and local-file serving helpers. */
import { open, readFile, stat, unlink } from 'fs/promises';
import { randomUUID } from 'crypto';
import { join, resolve as resolvePath } from 'path';
import { MIRRORED_MIME_TYPES } from './beeperAttachmentPaths.js';
import { ServerError } from './errorHandler.js';
import { copyFileGuarded, ensureDir, pathExists, renameGuarded, unlinkGuarded, writeFileGuarded } from './fileCore.js';
import {
  detectImageFormat, detectMediaManifest, getFileExtension, getMimeType, id3v2TagLength,
  MEDIA_MANIFEST_PROBE_BYTES, RISKY_MIME_TYPES, sanitizeFilename,
} from './mimeTypes.js';
import { PATHS } from './paths.js';
import { assertSafeFilename, isPathInsideDir } from './pathSafety.js';

// Uploads under these types may later be probed, thumbnailed or rendered by ffmpeg.
const isFfmpegInputExtension = (ext) => /^(audio|video|image)\//.test(getMimeType(ext));

/**
 * Refuse bytes ffmpeg would open as a playlist or manifest naming other files
 * (see `detectMediaManifest`). Apply before persisting any upload a probe,
 * thumbnail or render may later read.
 */
function assertNotMediaManifest(buffer) {
  const format = detectMediaManifest(buffer);
  if (format) {
    throw new ServerError(`File content is a ${format} playlist or manifest that references other files; upload the media itself`, { status: 400, code: 'INVALID_FILE_TYPE' });
  }
}

/**
 * `assertNotMediaManifest` over a file on disk: its leading window, plus a
 * full window after an ID3v2 tag. A tag ending inside a full leading window
 * leaves only a fragment of the following manifest in it, so the post-tag
 * window is read whenever the leading read filled its buffer.
 */
export async function assertFileNotMediaManifest(path) {
  const handle = await open(path, 'r');
  const readWindow = async (position) => {
    const window = Buffer.alloc(MEDIA_MANIFEST_PROBE_BYTES);
    const { bytesRead } = await handle.read(window, 0, window.length, position);
    return window.subarray(0, bytesRead);
  };
  const windows = await (async () => {
    const head = await readWindow(0);
    const tagLength = id3v2TagLength(head);
    return tagLength > 0 && head.length === MEDIA_MANIFEST_PROBE_BYTES ? [head, await readWindow(tagLength)] : [head];
  })().finally(() => handle.close());
  for (const window of windows) assertNotMediaManifest(window);
}

/**
 * Move a temp file into `destDir` and return the persisted filename, using the
 * `/api/uploads` route's naming (`<uuid8>-<sanitized-name>`) so a
 * server-produced file is served and referenced exactly like a user upload.
 *
 * copyFile + unlink instead of rename — the source usually lives in
 * `os.tmpdir()`, which may sit on a different filesystem (rename across devices
 * throws EXDEV on Linux); copy works regardless. The temp unlink is best-effort
 * cleanup. When `extensions` is supplied, the persisted name must use one of
 * those extensions. A persisted extension determines the Content-Type for
 * files that may live under a static mount, so it must never come from a
 * client-supplied filename. Returns `{ filename, sizeBytes }`.
 */
export async function importFileToDir(tempPath, originalName, destDir, { extensions } = {}) {
  const extension = getFileExtension(originalName);
  if (extensions && !extensions.map((item) => item.toLowerCase()).includes(extension)) {
    throw new ServerError('Invalid file type', { status: 400, code: 'INVALID_FILE_TYPE' });
  }
  if (extension && isFfmpegInputExtension(extension)) await assertFileNotMediaManifest(tempPath);
  await ensureDir(destDir);
  const id = randomUUID();
  const filename = `${id.slice(0, 8)}-${sanitizeFilename(originalName)}`;
  const dest = join(destDir, filename);
  await copyFileGuarded(tempPath, dest);
  await unlink(tempPath).catch(() => {});
  const s = await stat(dest).catch(() => null);
  return { id, filename, sizeBytes: s?.size ?? 0 };
}

/**
 * `importFileToDir` pinned to the uploads dir (`PATHS.uploads`) — the common
 * case (e.g. a yt-dlp audio extraction, #2120).
 */
export async function importFileToUploads(tempPath, originalName) {
  return importFileToDir(tempPath, originalName, PATHS.uploads);
}

// Widest supported image signature (WebP needs 12 bytes: `RIFF….WEBP`). A payload
// shorter than this can't hold a complete header for any format we accept.
const MIN_IMAGE_BYTES = 12;

// POSIX NAME_MAX — the per-component filename limit on ext4/APFS/HFS+. A longer
// component makes the write fail with ENAMETOOLONG.
const MAX_FILENAME_BYTES = 255;

/**
 * Persist a base64-encoded IMAGE into `dir`, deriving the format from the bytes
 * rather than trusting the client: decode → size cap → magic-byte sniff → force
 * the extension to the detected format → sanitize + containment guard → write.
 *
 * This is the sibling of `saveBase64Upload` for the paths that must be certain
 * they wrote an image (they hand the file to an image-gen backend or to an agent
 * that will read it), so an extension allowlist isn't enough. Naming is the
 * CALLER's call — pass a name that is already unique if later uploads must not
 * overwrite this one, since `dir` is a shared bucket. The bytes are staged in a
 * short-named sibling file and renamed into place, so a write that fails midway
 * (ENOSPC, …) never truncates an image already stored under the final name.
 * `stagePrefix` lets an owner keep its recovery prefix on the stage name (the
 * Persistent Mind sweeps `mind-<attachmentId>-*` images when a cleanup fails).
 *
 * Throws ServerError with the status/code/message contract `routes/screenshots.js`
 * established (`FILE_TOO_LARGE`, `INVALID_FILE_TYPE`, `INVALID_FILENAME` — all
 * 400) so callers keep their pinned responses.
 *
 * @param {string} dir - Destination directory (created if missing).
 * @param {{ filename: string, data: string }} upload - Desired base name (the
 *   extension is replaced with the detected one) + base64 payload.
 * @param {{ maxBytes: number, stagePrefix?: string }} opts
 * @returns {Promise<{ filename: string, filePath: string, size: number,
 *   format: string, mime: string }>}
 */
export async function saveImageUpload(dir, { filename, data }, { maxBytes, stagePrefix = '' }) {
  const buffer = Buffer.from(data, 'base64');
  if (buffer.length > maxBytes) {
    throw new ServerError(`File exceeds maximum size of ${maxBytes / 1024 / 1024}MB`, { status: 400, code: 'FILE_TOO_LARGE' });
  }

  // Floor before the sniff, because the per-format signatures are short (a JPEG is
  // 3 bytes, a GIF header 6) and would happily "detect" a truncated payload — which
  // then saves as a success and hands a consumer a path to an unreadable file. 12
  // bytes is the widest signature (WebP's RIFF….WEBP), so anything under it cannot
  // be a complete header for ANY supported format, let alone a complete image.
  if (buffer.length < MIN_IMAGE_BYTES) {
    throw new ServerError('Invalid image file - only PNG, JPEG, GIF, and WebP are supported', { status: 400, code: 'INVALID_FILE_TYPE' });
  }

  const detected = detectImageFormat(buffer);
  if (!detected) {
    throw new ServerError('Invalid image file - only PNG, JPEG, GIF, and WebP are supported', { status: 400, code: 'INVALID_FILE_TYPE' });
  }

  // The DETECTED extension always wins over whatever the client claimed, so the
  // file on disk can't advertise a type its bytes contradict. Strip a matching
  // extension off the base rather than appending a second one, then clamp the base
  // so `base + ext` fits NAME_MAX — a caller that prefixes the name (see
  // services/shellImageDrop.js) can otherwise push a legitimately-long client
  // filename past the limit and turn the write into an ENAMETOOLONG 500.
  const safeName = sanitizeFilename(filename);
  const base = safeName.toLowerCase().endsWith(detected.ext)
    ? safeName.slice(0, -detected.ext.length)
    : safeName;
  // sanitizeFilename leaves only ASCII, so slicing chars is slicing bytes.
  const fname = `${base.slice(0, MAX_FILENAME_BYTES - detected.ext.length)}${detected.ext}`;
  const filePath = join(dir, fname);
  if (!isPathInsideDir(dir, filePath)) {
    throw new ServerError('Invalid filename', { status: 400, code: 'INVALID_FILENAME' });
  }

  await ensureDir(dir);
  // Stage + rename rather than atomicWrite: the stored name is clamped to
  // NAME_MAX just above, and atomicWrite's `.<pid>.<ts>.<uuid>.tmp` suffix would
  // push the temp file past it (#6176). The stage name is short and fixed-size
  // whatever the final name is. Both paths are in `dir`, so rename is atomic.
  const stagePath = join(dir, `${stagePrefix}.stage-${randomUUID().slice(0, 8)}${detected.ext}`);
  if (!isPathInsideDir(dir, stagePath)) {
    throw new ServerError('Invalid filename', { status: 400, code: 'INVALID_FILENAME' });
  }
  const published = await writeFileGuarded(stagePath, buffer)
    .then(() => renameGuarded(stagePath, filePath))
    .then(() => null, (error) => error);
  if (published) {
    // Best-effort: a stage that survives keeps its owner prefix for sweeps.
    await unlinkGuarded(stagePath).catch(() => {});
    throw published;
  }

  return { filename: fname, filePath, size: buffer.length, format: detected.format, mime: detected.mime };
}

/**
 * Persist a base64-encoded upload into `dir` with the shared attachment
 * pipeline: extension allowlist → base64 decode → size cap → ensureDir →
 * `<uuid8>-<sanitized-name>` naming → containment guard → write.
 *
 * Throws ServerError with the exact status/code/message contract the
 * attachment routes established (`INVALID_FILE_TYPE`, `FILE_TOO_LARGE`,
 * `INVALID_FILENAME` — all 400), so refactored routes keep their pinned
 * responses. Consolidates `routes/attachments.js`, `routes/brainSongbook.js`,
 * and `routes/uploads.js` — the last keeps its own richer response shape
 * (`originalName` / `sizeFormatted` / `createdAt`) around these fields.
 *
 * @param {string} dir - Destination directory (created if missing).
 * @param {{ filename: string, data: string }} upload - Original filename +
 *   base64 payload (validated as present by the calling route).
 * @param {{ allowedExtensions: Set<string>, maxBytes: number }} opts
 * @returns {Promise<{ id: string, filename: string, filePath: string,
 *   buffer: Buffer, size: number, mime: string }>} `id` is the full UUID whose
 *   first 8 chars prefix `filename`.
 */
export async function saveBase64Upload(dir, { filename, data }, { allowedExtensions, maxBytes }) {
  const ext = getFileExtension(filename);
  if (!ext || !allowedExtensions.has(ext)) {
    const allowedList = [...allowedExtensions].join(', ');
    throw new ServerError(`File type not allowed. Supported: ${allowedList}`, { status: 400, code: 'INVALID_FILE_TYPE' });
  }

  const buffer = Buffer.from(data, 'base64');
  if (buffer.length > maxBytes) {
    throw new ServerError(`File exceeds maximum size of ${maxBytes / 1024 / 1024}MB`, { status: 400, code: 'FILE_TOO_LARGE' });
  }
  if (isFfmpegInputExtension(ext)) assertNotMediaManifest(buffer);

  await ensureDir(dir);

  // Unique uuid prefix avoids collisions; sanitize kills traversal characters.
  const id = randomUUID();
  const fname = `${id.slice(0, 8)}-${sanitizeFilename(filename)}`;
  const filePath = join(dir, fname);
  // Double-check the path is within the destination dir (defense in depth).
  if (!isPathInsideDir(dir, filePath)) {
    throw new ServerError('Invalid filename', { status: 400, code: 'INVALID_FILENAME' });
  }

  await writeFileGuarded(filePath, buffer);

  return { id, filename: fname, filePath, buffer, size: buffer.length, mime: getMimeType(getFileExtension(fname)) };
}

/**
 * A declared media type reduced to a bare, comparable `type/subtype`:
 * parameters dropped, whitespace trimmed on BOTH sides of the split (a value
 * like ` text/html ;charset=utf-8` leaves a trailing space otherwise), case
 * folded. Returns `''` for anything that is not a non-empty string.
 */
function normalizeDeclaredMimeType(value) {
  return typeof value === 'string' ? value.split(';')[0].trim().toLowerCase() : '';
}

/**
 * Serve a machine-local file from `dir` by user-supplied filename with the
 * shared safety pipeline: sanitize → containment guard (400
 * `INVALID_FILENAME`) → existence check (404, message/code parametrized via
 * `missingError` so routes keep their contracts) → `X-Content-Type-Options:
 * nosniff` → attachment disposition for RISKY_MIME_TYPES → `res.sendFile`.
 *
 * Mirrors the exact serving behavior of `routes/attachments.js`.
 *
 * `contentType` overrides the extension-derived type for a store whose
 * filenames carry no meaning. The Beeper attachment mirror (#37) is the
 * caller: it is content-addressed (`<sha256>.<ext>`), and the source it
 * downloads from (`GET /v1/assets/serve`) sends no `Content-Type` header at
 * all, so the type Beeper declared in the message payload is the only real
 * answer and the extension is decoration.
 *
 * That declared type is written by a REMOTE SENDER and this route is on the
 * authenticated dashboard origin, so the override is normalized and bounded
 * rather than trusted:
 *
 *   - Normalized first — parameters dropped, trimmed, lowercased. A media type
 *     is case-insensitive (RFC 2045) while `RISKY_MIME_TYPES` is a lowercase
 *     set, so `text/HTML` walked straight past the guard until this step
 *     existed and executed inline on the dashboard's own origin.
 *   - Bounded second — only a type the mirror itself can produce
 *     (`MIRRORED_MIME_TYPES`) is echoed; anything else is served as
 *     `application/octet-stream` instead of whatever was declared.
 *
 * The disposition decision is then taken on the declared type as well as the
 * served one, so downgrading a `text/html` to octet-stream never costs it the
 * attachment header it earned.
 *
 * @param {import('express').Response} res
 * @param {string} dir - The containing directory.
 * @param {string} filename - Raw user-supplied filename.
 * @param {{ missingError?: { message: string, code: string }, contentType?: string }} [opts]
 */
export async function serveLocalFile(res, dir, filename, { missingError, contentType } = {}) {
  const safeFilename = sanitizeFilename(filename);
  const filePath = resolvePath(dir, safeFilename);

  if (!isPathInsideDir(dir, filePath)) {
    throw new ServerError('Invalid filename', { status: 400, code: 'INVALID_FILENAME' });
  }

  if (!(await pathExists(filePath))) {
    const { message = 'Attachment not found', code = 'NOT_FOUND' } = missingError || {};
    throw new ServerError(message, { status: 404, code });
  }

  const declared = normalizeDeclaredMimeType(contentType);
  const mimeType = declared
    ? (MIRRORED_MIME_TYPES.has(declared) ? declared : 'application/octet-stream')
    : getMimeType(getFileExtension(safeFilename));
  res.set('X-Content-Type-Options', 'nosniff');
  if (RISKY_MIME_TYPES.has(mimeType) || RISKY_MIME_TYPES.has(declared)) {
    res.set('Content-Disposition', `attachment; filename="${safeFilename}"`);
  }
  res.type(mimeType).sendFile(filePath);
}