Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Streaming multipart/form-data parser — multer.diskStorage() replacement.
*
* True streaming: file content is written to disk in chunks as it arrives,
* never buffered in memory. Text fields are buffered (small by definition).
* Boundary detection keeps a small lookback (boundary length) between
* chunks so a boundary spanning a chunk edge isn't missed.
*
* Returns an Express middleware. Populates:
* - req.body[name] — for text parts (Content-Disposition has no filename)
* - req.file — uploadSingle: { path, originalname, mimetype, size }
* for the matching file part. Other file parts (different
* field names) are silently skipped.
* - req.files — uploadFields: { [fieldName]: { path, ... } } for each
* accepted file field. Field names not in the accepted
* set are silently skipped.
*/
import { createWriteStream } from 'fs';
import { unlink } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { randomUUID } from 'crypto';
const parseBoundary = (req) => {
const ct = req.headers['content-type'] || '';
// Media type is case-insensitive (RFC 2045), but the boundary value is
// case-sensitive — match the type prefix on a lowercased copy and parse
// the boundary off the original.
if (!ct.toLowerCase().startsWith('multipart/form-data')) {
const err = new Error('Expected multipart/form-data');
err.status = 400; err.code = 'INVALID_CONTENT_TYPE';
return { err };
}
const bm = ct.match(/boundary=([^\s;]+)/i);
if (!bm) {
const err = new Error('Missing multipart boundary');
err.status = 400; err.code = 'INVALID_CONTENT_TYPE';
return { err };
}
return { boundary: bm[1] };
};
export function uploadSingle(fieldName, { limits = {}, fileFilter } = {}) {
const maxSize = limits.fileSize ?? Infinity;
const accepted = new Set([fieldName]);
return (req, res, next) => {
const { err, boundary } = parseBoundary(req);
if (err) return next(err);
streamMultipart(req, boundary, accepted, maxSize, fileFilter, (innerErr) => {
if (innerErr) return next(innerErr);
// Back-compat: surface the matched file as req.file for callers that
// predate the multi-file `req.files` shape.
if (req.files?.[fieldName]) req.file = req.files[fieldName];
next();
});
};
}
// Multi-file variant — accepts a list of allowed file field names and
// populates req.files = { [fieldName]: { path, originalname, mimetype, size } }
// for each one that arrives. Same semantics as uploadSingle for everything
// else (text fields → req.body, unknown file fields silently dropped).
export function uploadFields(fieldNames, { limits = {}, fileFilter } = {}) {
const maxSize = limits.fileSize ?? Infinity;
const accepted = new Set(fieldNames);
return (req, res, next) => {
const { err, boundary } = parseBoundary(req);
if (err) return next(err);
streamMultipart(req, boundary, accepted, maxSize, fileFilter, next);
};
}
// Wrap uploadSingle so the parser only kicks in for multipart bodies — JSON
// callers fall through untouched. Useful when an existing JSON endpoint adds
// optional file-upload support without breaking back-compat.
export function optionalUpload(fieldName, opts) {
const uploader = uploadSingle(fieldName, opts);
return (req, res, next) => {
const ct = req.headers['content-type'] || '';
if (!ct.toLowerCase().startsWith('multipart/form-data')) return next();
return uploader(req, res, next);
};
}
// Multi-field version of optionalUpload — same JSON-body passthrough, but
// accepts the uploadFields field-name list. Files land on req.files keyed by
// field name; non-multipart JSON requests skip the parser entirely.
export function optionalUploadFields(fieldNames, opts) {
const uploader = uploadFields(fieldNames, opts);
return (req, res, next) => {
const ct = req.headers['content-type'] || '';
if (!ct.toLowerCase().startsWith('multipart/form-data')) return next();
return uploader(req, res, next);
};
}
function streamMultipart(req, boundary, acceptedNames, maxSize, fileFilter, next) {
const PART_DELIM = Buffer.from('\r\n--' + boundary);
const FIRST_DELIM = Buffer.from('--' + boundary);
const HEADER_END = Buffer.from('\r\n\r\n');
const STATE_PREAMBLE = 0; // before first boundary
const STATE_HEADERS = 1; // accumulating part headers
const STATE_BODY = 2; // streaming part body (text or file)
const STATE_AFTER_BOUNDARY = 3; // just past a boundary; expect CRLF or `--`
const STATE_DONE = 4;
let state = STATE_PREAMBLE;
let buf = Buffer.alloc(0);
let done = false;
let pendingFlush = 0; // outstanding async ws.end callbacks
let endSeen = false; // 'end' event fired but we may still be flushing
let textCharCount = 0;
const TEXT_FIELD_TOTAL_CAP = 1024 * 1024; // 1MB cap on aggregate text fields
// Cap the bytes we'll silently consume from non-matching file parts.
// Without this, a client could keep the connection busy by streaming a huge
// file under an unaccepted field name (we just discard the bytes, but the
// parser would happily read forever). Tie the cap to the endpoint's own
// per-file `maxSize` so an endpoint configured for 1MB uploads doesn't
// accept ~50MB of dropped bytes before 413-ing — the effective DoS surface
// matches the declared upload limit. Endpoints with no limit (Infinity)
// get a safety default so the dropped-bytes path isn't itself unbounded.
let droppedFileBytes = 0;
const DROPPED_FILE_TOTAL_CAP = Number.isFinite(maxSize) ? maxSize : 50 * 1024 * 1024;
// Per-part state
let currentName = null;
let currentFilename = null;
let isMatchingFile = false; // true when current part is one of acceptedNames
let writeStream = null;
let writePath = null;
let bytesWritten = 0;
let textBuf = null; // Buffer when current part is a text field
const body = {};
// Map of finalized files keyed by field name. Truncated-request cleanup
// walks this to unlink everything that was already flushed before the
// request died mid-stream.
const fileResults = {};
const fail = (err) => {
if (done) return;
done = true;
state = STATE_DONE;
if (writeStream) {
writeStream.destroy();
// Best-effort cleanup of the partially-written file.
if (writePath) unlink(writePath).catch(() => {});
}
// Also drop any already-finalized files — a fail mid-request must not
// leak completed uploads to the OS temp dir.
for (const f of Object.values(fileResults)) {
if (f?.path) unlink(f.path).catch(() => {});
}
req.removeAllListeners('data');
next(err);
};
const finish = () => {
if (done) return;
// If 'end' fires before we hit the terminal `--<boundary>--`, the
// request is truncated/malformed — reject with 400 instead of silently
// accepting a partial upload (could matter for a 2GB Apple Health
// import that gets cut off mid-stream).
if (state !== STATE_DONE) {
// Clean up any partially-written file before failing.
if (writeStream) writeStream.destroy();
if (writePath) unlink(writePath).catch(() => {});
// Also drop any files already finalized before the truncation.
for (const f of Object.values(fileResults)) {
if (f?.path) unlink(f.path).catch(() => {});
}
done = true;
const err = new Error('Truncated multipart request — never reached terminal boundary');
err.code = 'INVALID_MULTIPART';
err.status = 400;
next(err);
return;
}
done = true;
req.body = body;
if (Object.keys(fileResults).length) req.files = fileResults;
next();
};
// Start a new part — parse headers, set up text buffer or file write stream.
const startPart = (headerBlock) => {
// Reset per-part state so a part with no Content-Type can't inherit
// the previous part's mimetype, and stale flags don't carry over.
currentName = null;
currentFilename = null;
currentFileMimetype = null;
isMatchingFile = false;
textBuf = null;
const headerStr = headerBlock.toString('utf-8');
// Negative lookbehind keeps `name=` inside `filename=` from matching.
const nameMatch = headerStr.match(/(?<!file)name="([^"]+)"/i);
if (!nameMatch) return fail(new Error('Multipart part missing Content-Disposition name'));
currentName = nameMatch[1];
const filenameMatch = headerStr.match(/filename="([^"]*)"/i);
currentFilename = filenameMatch?.[1];
if (currentFilename != null && currentFilename !== '') {
isMatchingFile = acceptedNames.has(currentName);
const mimeMatch = headerStr.match(/Content-Type:\s*([^\r\n]+)/i);
currentFileMimetype = mimeMatch ? mimeMatch[1].trim() : 'application/octet-stream';
if (isMatchingFile) {
// `fieldname` matches multer's API contract — videoGen.js's filter
// dispatches on it to enforce per-field MIME rules (image/* on
// sourceImage|lastImage, audio/* on audioFile).
const fileMeta = { fieldname: currentName, originalname: currentFilename, mimetype: currentFileMimetype };
if (fileFilter) {
// CONTRACT: fileFilter MUST call its `cb` synchronously. We read
// `fr` immediately because the parser is consuming a streaming
// body and can't pause headers indefinitely without buffering
// unbounded bytes for the next part. (multer's docs allow async
// filters, but in this codebase all callers — appleHealth.js
// and videoGen.js — are sync. We surface a clear error if a
// future async filter slips in instead of crashing on `fr.err`
// dereference.)
let fr = null;
fileFilter(req, fileMeta, (err, accept) => { fr = { err, accept }; });
if (fr === null) {
const err = new Error('fileFilter must invoke its callback synchronously');
err.status = 500; err.code = 'INVALID_FILE_FILTER';
return fail(err);
}
if (fr.err) return fail(fr.err);
if (!fr.accept) {
// Set status/code so the centralized error middleware returns a
// 400 to the client instead of a 500 (the err thrown by the
// filter rejection is a client-input error, not a server bug).
const rejectErr = new Error('File type not allowed');
rejectErr.status = 400;
rejectErr.code = 'INVALID_FILE_TYPE';
return fail(rejectErr);
}
}
const rawExt = currentFilename.match(/\.[^.]+$/)?.[0] || '';
const ext = rawExt.replace(/[^a-zA-Z0-9.]/g, '');
writePath = join(tmpdir(), `upload-${randomUUID()}${ext}`);
writeStream = createWriteStream(writePath);
writeStream.on('error', fail);
bytesWritten = 0;
}
// Non-matching file part: no buffer, no stream — bytes are silently
// discarded as the body advances past them.
} else {
textBuf = Buffer.alloc(0);
}
};
// Emit `chunk` to the current part's destination (file write or text buffer
// or the void). Enforces maxSize for the matching file and the global
// text-field cap. Returns true to keep streaming, false on error/abort.
const writePartChunk = (chunk) => {
if (chunk.length === 0) return true;
if (currentFilename != null && currentFilename !== '') {
if (isMatchingFile) {
bytesWritten += chunk.length;
if (bytesWritten > maxSize) {
const err = new Error(`File too large (max ${maxSize} bytes)`);
err.status = 413; err.code = 'PAYLOAD_TOO_LARGE';
fail(err);
return false;
}
if (!writeStream.write(chunk)) {
// Backpressure: pause the request stream and resume on drain.
req.pause();
writeStream.once('drain', () => req.resume());
}
} else {
// Non-matching file: drop bytes, but cap the total so a client
// can't tie up the connection by sending an unbounded payload
// under an unaccepted field name.
droppedFileBytes += chunk.length;
if (droppedFileBytes > DROPPED_FILE_TOTAL_CAP) {
const err = new Error(`Unaccepted file part too large (max ${DROPPED_FILE_TOTAL_CAP} bytes)`);
err.status = 413; err.code = 'PAYLOAD_TOO_LARGE';
fail(err);
return false;
}
}
} else {
textCharCount += chunk.length;
if (textCharCount > TEXT_FIELD_TOTAL_CAP) {
const err = new Error(`Text fields too large (max ${TEXT_FIELD_TOTAL_CAP} bytes)`);
err.status = 413; err.code = 'PAYLOAD_TOO_LARGE';
fail(err);
return false;
}
textBuf = Buffer.concat([textBuf, chunk]);
}
return true;
};
// Finalize the current part — flush any pending data, then move on.
const endPart = (cb) => {
if (currentFilename != null && currentFilename !== '') {
if (isMatchingFile && writeStream) {
const ws = writeStream;
const path = writePath;
const size = bytesWritten;
const filename = currentFilename;
const mimetype = currentFileMimetype || 'application/octet-stream';
const fieldName = currentName;
writeStream = null;
writePath = null;
pendingFlush += 1;
ws.end(() => {
// Defensive: a malformed/malicious request can include the same
// field name twice. Drop the prior temp file before overwriting
// so it doesn't leak in os.tmpdir().
const prior = fileResults[fieldName];
if (prior?.path) unlink(prior.path).catch(() => {});
fileResults[fieldName] = { path, originalname: filename, mimetype, size };
pendingFlush -= 1;
cb();
// If 'end' arrived while we were still flushing, finalize now.
if (endSeen && pendingFlush === 0 && !done) finish();
});
} else {
cb();
}
} else if (textBuf != null) {
const value = textBuf.toString('utf-8');
if (Object.prototype.hasOwnProperty.call(body, currentName)) {
if (Array.isArray(body[currentName])) body[currentName].push(value);
else body[currentName] = [body[currentName], value];
} else {
body[currentName] = value;
}
textBuf = null;
cb();
} else {
cb();
}
};
// Captured by startPart so endPart's async cb has the right mimetype even
// after the next part may have started.
let currentFileMimetype = null;
const tick = () => {
if (done) return;
// Defer while a file flush is in flight: state is still STATE_BODY but
// writeStream has been nulled by endPart. Re-entering tick (e.g. from
// req.on('end')) would crash on writeStream.write. The endPart callback
// re-invokes tick once state has advanced.
if (pendingFlush > 0) return;
while (true) {
if (state === STATE_PREAMBLE) {
const idx = buf.indexOf(FIRST_DELIM);
if (idx === -1) {
// Need more bytes; drop everything past the last possible boundary start.
const safe = buf.length - (FIRST_DELIM.length - 1);
if (safe > 0) buf = buf.slice(safe);
return;
}
buf = buf.slice(idx + FIRST_DELIM.length);
state = STATE_AFTER_BOUNDARY;
continue;
}
if (state === STATE_AFTER_BOUNDARY) {
if (buf.length < 2) return;
const trailing = buf.slice(0, 2);
if (trailing[0] === 0x2d && trailing[1] === 0x2d) { // `--`
buf = Buffer.alloc(0);
// Mark clean termination so finish() can distinguish a proper
// end from a truncated request that hit 'end' mid-stream.
state = STATE_DONE;
return finish();
}
if (trailing[0] !== 0x0d || trailing[1] !== 0x0a) {
const err = new Error('Malformed multipart: missing CRLF after boundary');
err.status = 400; err.code = 'INVALID_MULTIPART';
return fail(err);
}
buf = buf.slice(2);
state = STATE_HEADERS;
continue;
}
if (state === STATE_HEADERS) {
const hEnd = buf.indexOf(HEADER_END);
if (hEnd === -1) return; // wait for more bytes
const headers = buf.slice(0, hEnd);
buf = buf.slice(hEnd + HEADER_END.length);
startPart(headers);
if (done) return;
state = STATE_BODY;
continue;
}
if (state === STATE_BODY) {
const idx = buf.indexOf(PART_DELIM);
if (idx === -1) {
// No boundary yet — emit what's safe and hold back PART_DELIM.length-1
// trailing bytes so a boundary spanning the next chunk isn't missed.
const safe = buf.length - (PART_DELIM.length - 1);
if (safe > 0) {
if (!writePartChunk(buf.slice(0, safe))) return;
buf = buf.slice(safe);
}
return;
}
// Found end of part. Emit everything before the boundary, then advance.
if (!writePartChunk(buf.slice(0, idx))) return;
buf = buf.slice(idx + PART_DELIM.length);
// Transition state SYNCHRONOUSLY before endPart's async ws.end() runs.
// endPart sets writeStream = null synchronously, but its file-flush
// callback (which used to set this state) may not fire before
// req.on('end') re-enters tick. Without this sync transition, tick
// would re-process buf (which now contains the NEXT part) as if we
// were still inside the previous part's body and crash on
// writeStream.write(chunk) with writeStream === null.
state = STATE_AFTER_BOUNDARY;
// endPart may still be async (file flush) — pause the request,
// resume after the flush completes.
req.pause();
endPart(() => {
req.resume();
tick(); // process anything already buffered
});
return;
}
return; // unknown state
}
};
req.on('data', (chunk) => {
if (done) return;
buf = Buffer.concat([buf, chunk]);
tick();
});
req.on('end', () => {
if (done) return;
endSeen = true;
// Drain anything still in buf as part of the current state.
tick();
// If a file flush is still pending, the endPart callback will finalize.
if (!done && pendingFlush === 0) finish();
});
req.on('error', fail);
}