Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

18.2 KB
Raw Blame
/**
 * Streaming multipart/form-data parser — multer.diskStorage() replacement.
 *
 * True streaming: file content is written to disk in chunks as it arrives,
 * never buffered in memory. Text fields are buffered (small by definition).
 * Boundary detection keeps a small lookback (boundary length) between
 * chunks so a boundary spanning a chunk edge isn't missed.
 *
 * Returns an Express middleware. Populates:
 *   - req.body[name]  — for text parts (Content-Disposition has no filename)
 *   - req.file        — uploadSingle: { path, originalname, mimetype, size }
 *                       for the matching file part. Other file parts (different
 *                       field names) are silently skipped.
 *   - req.files       — uploadFields: { [fieldName]: { path, ... } } for each
 *                       accepted file field. Field names not in the accepted
 *                       set are silently skipped.
 */

import { createWriteStream } from 'fs';
import { unlink } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { randomUUID } from 'crypto';

const parseBoundary = (req) => {
  const ct = req.headers['content-type'] || '';
  // Media type is case-insensitive (RFC 2045), but the boundary value is
  // case-sensitive — match the type prefix on a lowercased copy and parse
  // the boundary off the original.
  if (!ct.toLowerCase().startsWith('multipart/form-data')) {
    const err = new Error('Expected multipart/form-data');
    err.status = 400; err.code = 'INVALID_CONTENT_TYPE';
    return { err };
  }
  const bm = ct.match(/boundary=([^\s;]+)/i);
  if (!bm) {
    const err = new Error('Missing multipart boundary');
    err.status = 400; err.code = 'INVALID_CONTENT_TYPE';
    return { err };
  }
  return { boundary: bm[1] };
};

export function uploadSingle(fieldName, { limits = {}, fileFilter } = {}) {
  const maxSize = limits.fileSize ?? Infinity;
  const accepted = new Set([fieldName]);

  return (req, res, next) => {
    const { err, boundary } = parseBoundary(req);
    if (err) return next(err);
    streamMultipart(req, boundary, accepted, maxSize, fileFilter, (innerErr) => {
      if (innerErr) return next(innerErr);
      // Back-compat: surface the matched file as req.file for callers that
      // predate the multi-file `req.files` shape.
      if (req.files?.[fieldName]) req.file = req.files[fieldName];
      next();
    });
  };
}

// Multi-file variant — accepts a list of allowed file field names and
// populates req.files = { [fieldName]: { path, originalname, mimetype, size } }
// for each one that arrives. Same semantics as uploadSingle for everything
// else (text fields → req.body, unknown file fields silently dropped).
export function uploadFields(fieldNames, { limits = {}, fileFilter } = {}) {
  const maxSize = limits.fileSize ?? Infinity;
  const accepted = new Set(fieldNames);

  return (req, res, next) => {
    const { err, boundary } = parseBoundary(req);
    if (err) return next(err);
    streamMultipart(req, boundary, accepted, maxSize, fileFilter, next);
  };
}

// Wrap uploadSingle so the parser only kicks in for multipart bodies — JSON
// callers fall through untouched. Useful when an existing JSON endpoint adds
// optional file-upload support without breaking back-compat.
export function optionalUpload(fieldName, opts) {
  const uploader = uploadSingle(fieldName, opts);
  return (req, res, next) => {
    const ct = req.headers['content-type'] || '';
    if (!ct.toLowerCase().startsWith('multipart/form-data')) return next();
    return uploader(req, res, next);
  };
}

// Multi-field version of optionalUpload — same JSON-body passthrough, but
// accepts the uploadFields field-name list. Files land on req.files keyed by
// field name; non-multipart JSON requests skip the parser entirely.
export function optionalUploadFields(fieldNames, opts) {
  const uploader = uploadFields(fieldNames, opts);
  return (req, res, next) => {
    const ct = req.headers['content-type'] || '';
    if (!ct.toLowerCase().startsWith('multipart/form-data')) return next();
    return uploader(req, res, next);
  };
}

function streamMultipart(req, boundary, acceptedNames, maxSize, fileFilter, next) {
  const PART_DELIM = Buffer.from('\r\n--' + boundary);
  const FIRST_DELIM = Buffer.from('--' + boundary);
  const HEADER_END = Buffer.from('\r\n\r\n');

  const STATE_PREAMBLE = 0;       // before first boundary
  const STATE_HEADERS = 1;        // accumulating part headers
  const STATE_BODY = 2;           // streaming part body (text or file)
  const STATE_AFTER_BOUNDARY = 3; // just past a boundary; expect CRLF or `--`
  const STATE_DONE = 4;

  let state = STATE_PREAMBLE;
  let buf = Buffer.alloc(0);
  let done = false;
  let pendingFlush = 0;       // outstanding async ws.end callbacks
  let endSeen = false;        // 'end' event fired but we may still be flushing
  let textCharCount = 0;
  const TEXT_FIELD_TOTAL_CAP = 1024 * 1024; // 1MB cap on aggregate text fields
  // Cap the bytes we'll silently consume from non-matching file parts.
  // Without this, a client could keep the connection busy by streaming a huge
  // file under an unaccepted field name (we just discard the bytes, but the
  // parser would happily read forever). Tie the cap to the endpoint's own
  // per-file `maxSize` so an endpoint configured for 1MB uploads doesn't
  // accept ~50MB of dropped bytes before 413-ing — the effective DoS surface
  // matches the declared upload limit. Endpoints with no limit (Infinity)
  // get a safety default so the dropped-bytes path isn't itself unbounded.
  let droppedFileBytes = 0;
  const DROPPED_FILE_TOTAL_CAP = Number.isFinite(maxSize) ? maxSize : 50 * 1024 * 1024;

  // Per-part state
  let currentName = null;
  let currentFilename = null;
  let isMatchingFile = false;     // true when current part is one of acceptedNames
  let writeStream = null;
  let writePath = null;
  let bytesWritten = 0;
  let textBuf = null;             // Buffer when current part is a text field

  const body = {};
  // Map of finalized files keyed by field name. Truncated-request cleanup
  // walks this to unlink everything that was already flushed before the
  // request died mid-stream.
  const fileResults = {};

  const fail = (err) => {
    if (done) return;
    done = true;
    state = STATE_DONE;
    if (writeStream) {
      writeStream.destroy();
      // Best-effort cleanup of the partially-written file.
      if (writePath) unlink(writePath).catch(() => {});
    }
    // Also drop any already-finalized files — a fail mid-request must not
    // leak completed uploads to the OS temp dir.
    for (const f of Object.values(fileResults)) {
      if (f?.path) unlink(f.path).catch(() => {});
    }
    req.removeAllListeners('data');
    next(err);
  };

  const finish = () => {
    if (done) return;
    // If 'end' fires before we hit the terminal `--<boundary>--`, the
    // request is truncated/malformed — reject with 400 instead of silently
    // accepting a partial upload (could matter for a 2GB Apple Health
    // import that gets cut off mid-stream).
    if (state !== STATE_DONE) {
      // Clean up any partially-written file before failing.
      if (writeStream) writeStream.destroy();
      if (writePath) unlink(writePath).catch(() => {});
      // Also drop any files already finalized before the truncation.
      for (const f of Object.values(fileResults)) {
        if (f?.path) unlink(f.path).catch(() => {});
      }
      done = true;
      const err = new Error('Truncated multipart request — never reached terminal boundary');
      err.code = 'INVALID_MULTIPART';
      err.status = 400;
      next(err);
      return;
    }
    done = true;
    req.body = body;
    if (Object.keys(fileResults).length) req.files = fileResults;
    next();
  };

  // Start a new part — parse headers, set up text buffer or file write stream.
  const startPart = (headerBlock) => {
    // Reset per-part state so a part with no Content-Type can't inherit
    // the previous part's mimetype, and stale flags don't carry over.
    currentName = null;
    currentFilename = null;
    currentFileMimetype = null;
    isMatchingFile = false;
    textBuf = null;
    const headerStr = headerBlock.toString('utf-8');
    // Negative lookbehind keeps `name=` inside `filename=` from matching.
    const nameMatch = headerStr.match(/(?<!file)name="([^"]+)"/i);
    if (!nameMatch) return fail(new Error('Multipart part missing Content-Disposition name'));
    currentName = nameMatch[1];
    const filenameMatch = headerStr.match(/filename="([^"]*)"/i);
    currentFilename = filenameMatch?.[1];

    if (currentFilename != null && currentFilename !== '') {
      isMatchingFile = acceptedNames.has(currentName);
      const mimeMatch = headerStr.match(/Content-Type:\s*([^\r\n]+)/i);
      currentFileMimetype = mimeMatch ? mimeMatch[1].trim() : 'application/octet-stream';
      if (isMatchingFile) {
        // `fieldname` matches multer's API contract — videoGen.js's filter
        // dispatches on it to enforce per-field MIME rules (image/* on
        // sourceImage|lastImage, audio/* on audioFile).
        const fileMeta = { fieldname: currentName, originalname: currentFilename, mimetype: currentFileMimetype };

        if (fileFilter) {
          // CONTRACT: fileFilter MUST call its `cb` synchronously. We read
          // `fr` immediately because the parser is consuming a streaming
          // body and can't pause headers indefinitely without buffering
          // unbounded bytes for the next part. (multer's docs allow async
          // filters, but in this codebase all callers — appleHealth.js
          // and videoGen.js — are sync. We surface a clear error if a
          // future async filter slips in instead of crashing on `fr.err`
          // dereference.)
          let fr = null;
          fileFilter(req, fileMeta, (err, accept) => { fr = { err, accept }; });
          if (fr === null) {
            const err = new Error('fileFilter must invoke its callback synchronously');
            err.status = 500; err.code = 'INVALID_FILE_FILTER';
            return fail(err);
          }
          if (fr.err) return fail(fr.err);
          if (!fr.accept) {
            // Set status/code so the centralized error middleware returns a
            // 400 to the client instead of a 500 (the err thrown by the
            // filter rejection is a client-input error, not a server bug).
            const rejectErr = new Error('File type not allowed');
            rejectErr.status = 400;
            rejectErr.code = 'INVALID_FILE_TYPE';
            return fail(rejectErr);
          }
        }

        const rawExt = currentFilename.match(/\.[^.]+$/)?.[0] || '';
        const ext = rawExt.replace(/[^a-zA-Z0-9.]/g, '');
        writePath = join(tmpdir(), `upload-${randomUUID()}${ext}`);
        writeStream = createWriteStream(writePath);
        writeStream.on('error', fail);
        bytesWritten = 0;
      }
      // Non-matching file part: no buffer, no stream — bytes are silently
      // discarded as the body advances past them.
    } else {
      textBuf = Buffer.alloc(0);
    }
  };

  // Emit `chunk` to the current part's destination (file write or text buffer
  // or the void). Enforces maxSize for the matching file and the global
  // text-field cap. Returns true to keep streaming, false on error/abort.
  const writePartChunk = (chunk) => {
    if (chunk.length === 0) return true;
    if (currentFilename != null && currentFilename !== '') {
      if (isMatchingFile) {
        bytesWritten += chunk.length;
        if (bytesWritten > maxSize) {
          const err = new Error(`File too large (max ${maxSize} bytes)`);
          err.status = 413; err.code = 'PAYLOAD_TOO_LARGE';
          fail(err);
          return false;
        }
        if (!writeStream.write(chunk)) {
          // Backpressure: pause the request stream and resume on drain.
          req.pause();
          writeStream.once('drain', () => req.resume());
        }
      } else {
        // Non-matching file: drop bytes, but cap the total so a client
        // can't tie up the connection by sending an unbounded payload
        // under an unaccepted field name.
        droppedFileBytes += chunk.length;
        if (droppedFileBytes > DROPPED_FILE_TOTAL_CAP) {
          const err = new Error(`Unaccepted file part too large (max ${DROPPED_FILE_TOTAL_CAP} bytes)`);
          err.status = 413; err.code = 'PAYLOAD_TOO_LARGE';
          fail(err);
          return false;
        }
      }
    } else {
      textCharCount += chunk.length;
      if (textCharCount > TEXT_FIELD_TOTAL_CAP) {
        const err = new Error(`Text fields too large (max ${TEXT_FIELD_TOTAL_CAP} bytes)`);
        err.status = 413; err.code = 'PAYLOAD_TOO_LARGE';
        fail(err);
        return false;
      }
      textBuf = Buffer.concat([textBuf, chunk]);
    }
    return true;
  };

  // Finalize the current part — flush any pending data, then move on.
  const endPart = (cb) => {
    if (currentFilename != null && currentFilename !== '') {
      if (isMatchingFile && writeStream) {
        const ws = writeStream;
        const path = writePath;
        const size = bytesWritten;
        const filename = currentFilename;
        const mimetype = currentFileMimetype || 'application/octet-stream';
        const fieldName = currentName;
        writeStream = null;
        writePath = null;
        pendingFlush += 1;
        ws.end(() => {
          // Defensive: a malformed/malicious request can include the same
          // field name twice. Drop the prior temp file before overwriting
          // so it doesn't leak in os.tmpdir().
          const prior = fileResults[fieldName];
          if (prior?.path) unlink(prior.path).catch(() => {});
          fileResults[fieldName] = { path, originalname: filename, mimetype, size };
          pendingFlush -= 1;
          cb();
          // If 'end' arrived while we were still flushing, finalize now.
          if (endSeen && pendingFlush === 0 && !done) finish();
        });
      } else {
        cb();
      }
    } else if (textBuf != null) {
      const value = textBuf.toString('utf-8');
      if (Object.prototype.hasOwnProperty.call(body, currentName)) {
        if (Array.isArray(body[currentName])) body[currentName].push(value);
        else body[currentName] = [body[currentName], value];
      } else {
        body[currentName] = value;
      }
      textBuf = null;
      cb();
    } else {
      cb();
    }
  };

  // Captured by startPart so endPart's async cb has the right mimetype even
  // after the next part may have started.
  let currentFileMimetype = null;

  const tick = () => {
    if (done) return;
    // Defer while a file flush is in flight: state is still STATE_BODY but
    // writeStream has been nulled by endPart. Re-entering tick (e.g. from
    // req.on('end')) would crash on writeStream.write. The endPart callback
    // re-invokes tick once state has advanced.
    if (pendingFlush > 0) return;

    while (true) {
      if (state === STATE_PREAMBLE) {
        const idx = buf.indexOf(FIRST_DELIM);
        if (idx === -1) {
          // Need more bytes; drop everything past the last possible boundary start.
          const safe = buf.length - (FIRST_DELIM.length - 1);
          if (safe > 0) buf = buf.slice(safe);
          return;
        }
        buf = buf.slice(idx + FIRST_DELIM.length);
        state = STATE_AFTER_BOUNDARY;
        continue;
      }

      if (state === STATE_AFTER_BOUNDARY) {
        if (buf.length < 2) return;
        const trailing = buf.slice(0, 2);
        if (trailing[0] === 0x2d && trailing[1] === 0x2d) { // `--`
          buf = Buffer.alloc(0);
          // Mark clean termination so finish() can distinguish a proper
          // end from a truncated request that hit 'end' mid-stream.
          state = STATE_DONE;
          return finish();
        }
        if (trailing[0] !== 0x0d || trailing[1] !== 0x0a) {
          const err = new Error('Malformed multipart: missing CRLF after boundary');
          err.status = 400; err.code = 'INVALID_MULTIPART';
          return fail(err);
        }
        buf = buf.slice(2);
        state = STATE_HEADERS;
        continue;
      }

      if (state === STATE_HEADERS) {
        const hEnd = buf.indexOf(HEADER_END);
        if (hEnd === -1) return; // wait for more bytes
        const headers = buf.slice(0, hEnd);
        buf = buf.slice(hEnd + HEADER_END.length);
        startPart(headers);
        if (done) return;
        state = STATE_BODY;
        continue;
      }

      if (state === STATE_BODY) {
        const idx = buf.indexOf(PART_DELIM);
        if (idx === -1) {
          // No boundary yet — emit what's safe and hold back PART_DELIM.length-1
          // trailing bytes so a boundary spanning the next chunk isn't missed.
          const safe = buf.length - (PART_DELIM.length - 1);
          if (safe > 0) {
            if (!writePartChunk(buf.slice(0, safe))) return;
            buf = buf.slice(safe);
          }
          return;
        }
        // Found end of part. Emit everything before the boundary, then advance.
        if (!writePartChunk(buf.slice(0, idx))) return;
        buf = buf.slice(idx + PART_DELIM.length);
        // Transition state SYNCHRONOUSLY before endPart's async ws.end() runs.
        // endPart sets writeStream = null synchronously, but its file-flush
        // callback (which used to set this state) may not fire before
        // req.on('end') re-enters tick. Without this sync transition, tick
        // would re-process buf (which now contains the NEXT part) as if we
        // were still inside the previous part's body and crash on
        // writeStream.write(chunk) with writeStream === null.
        state = STATE_AFTER_BOUNDARY;
        // endPart may still be async (file flush) — pause the request,
        // resume after the flush completes.
        req.pause();
        endPart(() => {
          req.resume();
          tick(); // process anything already buffered
        });
        return;
      }

      return; // unknown state
    }
  };

  req.on('data', (chunk) => {
    if (done) return;
    buf = Buffer.concat([buf, chunk]);
    tick();
  });

  req.on('end', () => {
    if (done) return;
    endSeen = true;
    // Drain anything still in buf as part of the current state.
    tick();
    // If a file flush is still pending, the endPart callback will finalize.
    if (!done && pendingFlush === 0) finish();
  });

  req.on('error', fail);
}