Skip to content

Open-source gym & body-weight tracker, now in the App Store (iPhone + Apple Watch) and as an Android APK. Plan routines, log workouts, see which muscles are trained, import from Strong/Hevy/FitNotes. Self-host it or use it on the phone alone.

API: a store for the photos and videos of custom exercises — per profile, named by their sha256, streamed to disk, and never deleted for anything missing

Duarte Santos424845df2ec3
diff --git a/api/media.js b/api/media.js
new file mode 100644
index 000000000..489227fbb
--- /dev/null
+++ b/api/media.js
@@ -0,0 +1,724 @@
+/* Photos, GIFs and videos of a person's own exercises: the server half.
+ *
+ * A custom exercise in the state carries a MediaRef (sha256, kind, mime, size, dimensions, a
+ * poster ref) and never the bytes. The bytes live here, per profile and named by their hash:
+ *
+ * DATA_DIR/uploads/<uid>/<sha256>.<ext> 0700 directories, 0600 files
+ * DATA_DIR/uploads/<uid>/.gc.json { hash: unreferencedSinceMs }
+ * DATA_DIR/uploads/<uid>/.tmp/<random> uploads in flight; never outlive their request
+ *
+ * What this module promises, and why each promise is shaped the way it is:
+ *
+ * - It never decodes media. The only bytes it interprets are the first 64 (magic numbers) and,
+ * for MP4/MOV, the box headers down to mvhd. No image library means no decompression bomb,
+ * no codec parser and no dependency; the metadata scrub and re-encode happen on the device
+ * that picked the file.
+ * - It never buffers an upload. The production API runs with a 128 MB memory limit and a video
+ * may be 40 MB, so bytes go from the socket through a hash to a temp file and nowhere else.
+ * - It never deletes because something is missing. A state file that cannot be read, a user
+ * that is not in db.json (a db.json that failed to parse boots with no users at all) and an
+ * unreadable .gc.json all mean "keep everything". Blobs only go when the profile's own
+ * readable state has not referenced them for the whole grace period, or when the profile
+ * itself is deleted. Devices that still hold a blob the server dropped upload it again.
+ * - The directory is the access check. Nothing here takes a uid from a request; every lookup
+ * is in the caller's own folder, so there is no way to learn whether someone else has a file.
+ */
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+
+export const HASH_RE = /^[0-9a-f]{64}$/;
+const MB = 1024 * 1024; // every *_MB setting means MiB; the client uses the same unit
+const HOUR = 3600000;
+const DAY = 24 * HOUR;
+const HEAD_BYTES = 64; // enough for every magic number below, the WebM DocType included
+const MOOV_MAX = 8 * MB; // a real moov for a 60 s clip is well under 1 MB
+const MAX_BOXES = 10000;
+export const MAX_INFLIGHT = 2; // concurrent uploads per profile
+
+/* The seven types that are ever stored. `category` is what the client may not lie about: a
+ JPEG declared as a GIF is still a still image and is stored as the JPEG it is, but a video
+ declared as an image would slip past the image cap, so that mismatch is refused. */
+export const MEDIA_TYPES = {
+ 'image/jpeg': { kind: 'image', category: 'still', ext: 'jpg' },
+ 'image/png': { kind: 'image', category: 'still', ext: 'png' },
+ 'image/webp': { kind: 'image', category: 'still', ext: 'webp' },
+ 'image/gif': { kind: 'gif', category: 'still', ext: 'gif' },
+ 'video/mp4': { kind: 'video', category: 'video', ext: 'mp4' },
+ 'video/quicktime': { kind: 'video', category: 'video', ext: 'mov' },
+ 'video/webm': { kind: 'video', category: 'video', ext: 'webm' }
+};
+const EXT_MIME = Object.fromEntries(Object.entries(MEDIA_TYPES).map(([mime, t]) => [t.ext, mime]));
+const FILE_RE = /^([0-9a-f]{64})\.(jpg|png|webp|gif|mp4|mov|webm)$/;
+
+/* ---------------------------------------------------------------- limits */
+
+const OFF = /^(0|false|no|off)$/i;
+/** The instance's media settings, read from the environment (docs/SELF_HOSTING.md). Every MB
+ * value is MiB and may be fractional. A value that does not parse falls back to its default
+ * rather than to 0: a typo must not turn the quota off or refuse every upload. */
+export function mediaLimits(env = process.env) {
+ const num = (key, def, min) => {
+ const raw = env[key];
+ if (raw == null || String(raw).trim() === '') return def;
+ const n = Number(raw);
+ return Number.isFinite(n) && n >= min ? n : def;
+ };
+ return {
+ enabled: !OFF.test(String(env.MEDIA_UPLOADS || '').trim()),
+ quotaMB: num('MEDIA_QUOTA_MB', 200, 0), // 0 = no cap
+ imageMB: num('MEDIA_IMAGE_MAX_MB', 2, 0.001),
+ gifMB: num('MEDIA_GIF_MAX_MB', 8, 0.001),
+ videoMB: num('MEDIA_VIDEO_MAX_MB', 40, 0.001),
+ videoSec: num('MEDIA_VIDEO_MAX_SEC', 60, 1),
+ gcGraceDays: num('MEDIA_GC_GRACE_DAYS', 14, 0),
+ uploadsPerHour: Math.floor(num('MEDIA_UPLOADS_PER_HOUR', 600, 1)),
+ minFreeMB: num('MEDIA_MIN_FREE_MB', 512, 0) // 0 = no floor
+ };
+}
+/** The block GET /api/config hands out. Public: the caps are not a secret, and the editor needs
+ * them before the first upload to refuse a file the server would refuse anyway. */
+export const mediaConfig = l => ({ imageMB: l.imageMB, gifMB: l.gifMB, videoMB: l.videoMB, videoSec: l.videoSec, quotaMB: l.quotaMB });
+
+/* ---------------------------------------------------------------- errors */
+
+const MESSAGES = {
+ 'bad-request': 'bad request',
+ 'media-type': 'that file type is not accepted',
+ 'media-too-large': 'that file is too large',
+ 'media-quota': 'your space for photos and videos is full',
+ 'media-invalid': 'that video could not be read',
+ 'media-too-long': 'that video is too long',
+ 'media-missing': 'no such file',
+ 'hash-mismatch': 'the file does not match its name',
+ 'storage-full': 'the server is running out of disk space',
+ busy: 'too many uploads at once — try again in a moment',
+ locked: 'too many uploads — try again later',
+ timeout: 'the upload stalled'
+};
+/** A refusal the client caused or can act on. The server's catch-all answers it as
+ * `{error, code, ...extra}` with its status and headers, and does not log it: a refused file
+ * is not a server error, and a stack trace per refusal would bury the real ones. */
+export class MediaError extends Error {
+ constructor(status, code, extra = {}, headers = undefined) {
+ super(MESSAGES[code] || code);
+ this.status = status;
+ this.code = code;
+ this.extra = extra;
+ this.headers = headers;
+ }
+}
+
+/* ---------------------------------------------------------------- sniffing */
+
+const MP4_BRANDS = new Set([
+ 'isom', 'iso2', 'iso3', 'iso4', 'iso5', 'iso6', 'mp41', 'mp42', 'avc1', 'M4V ', 'M4VP',
+ 'dash', 'mmp4', 'MSNV', '3gp4', '3gp5', '3gp6', '3g2a'
+]);
+
+// An EBML variable-length integer at b[i]. IDs keep their length marker, sizes drop it.
+function vint(b, i, keepMarker) {
+ if (i >= b.length || b[i] === 0) return null;
+ const first = b[i];
+ let len = 1;
+ while (!(first & (0x80 >> (len - 1)))) len++;
+ if (i + len > b.length) return null;
+ let v = keepMarker ? first : first & (0xff >> len);
+ for (let k = 1; k < len; k++) v = v * 256 + b[i + k];
+ return { value: v, len };
+}
+// The DocType of an EBML header ('webm', 'matroska'), read off the header's children.
+function ebmlDocType(b) {
+ const hs = vint(b, 4, false);
+ if (!hs) return null;
+ let i = 4 + hs.len;
+ const end = Math.min(b.length, i + hs.value);
+ while (i < end) {
+ const id = vint(b, i, true);
+ const sz = id && vint(b, i + id.len, false);
+ if (!sz) return null;
+ const body = i + id.len + sz.len;
+ if (id.value === 0x4282) {
+ if (body + sz.value > b.length) return null;
+ return Buffer.from(b.subarray(body, body + sz.value)).toString('latin1').replace(/\0+$/, '');
+ }
+ i = body + sz.value;
+ }
+ return null;
+}
+
+/**
+ * What a file is, from its first bytes alone — never from its name or a declared type.
+ * Returns { mime, kind, category, ext } or null. The client's sniffKind() follows the same rules
+ * and additionally reads HEIC/AVIF, which it only ever re-encodes; the server refuses them.
+ */
+export function sniffMedia(head) {
+ const b = Buffer.isBuffer(head) ? head : Buffer.from(head || []);
+ const as = mime => ({ mime, ...MEDIA_TYPES[mime] });
+ const at = (off, ...bytes) => bytes.every((x, i) => b[off + i] === x);
+ const ascii = (off, len) => b.length >= off + len ? b.toString('latin1', off, off + len) : '';
+ if (at(0, 0xff, 0xd8, 0xff)) return as('image/jpeg');
+ if (at(0, 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a)) return as('image/png');
+ if (ascii(0, 6) === 'GIF87a' || ascii(0, 6) === 'GIF89a') return as('image/gif');
+ if (ascii(0, 4) === 'RIFF' && ascii(8, 4) === 'WEBP') return as('image/webp');
+ if (ascii(4, 4) === 'ftyp') {
+ const brand = ascii(8, 4);
+ if (brand === 'qt ') return as('video/quicktime');
+ if (MP4_BRANDS.has(brand)) return as('video/mp4');
+ return null; // HEIC/AVIF stills, M4A audio and every brand nobody vouched for
+ }
+ if (at(0, 0x1a, 0x45, 0xdf, 0xa3)) return ebmlDocType(b) === 'webm' ? as('video/webm') : null;
+ return null;
+}
+
+/* ---------------------------------------------------------------- MP4 / MOV */
+
+function readAt(fd, pos, len) {
+ const buf = Buffer.alloc(len);
+ return fs.readSync(fd, buf, 0, len, pos) === len ? buf : null;
+}
+// The box whose header starts at `pos`, which must end by `end`. size 1 = a 64-bit size follows,
+// size 0 = the box runs to `end`. Anything that would reach outside its parent is a broken file.
+function boxAt(fd, pos, end) {
+ if (end - pos < 8) return null;
+ const h = readAt(fd, pos, Math.min(16, end - pos));
+ if (!h) return null;
+ let size = h.readUInt32BE(0);
+ let headerLen = 8;
+ if (size === 1) {
+ if (h.length < 16) return null;
+ size = u64(h, 8);
+ if (size === Number.MAX_SAFE_INTEGER) return null;
+ headerLen = 16;
+ } else if (size === 0) size = end - pos;
+ if (size < headerLen || pos + size > end) return null;
+ return { type: h.toString('latin1', 4, 8), body: pos + headerLen, bodyLen: size - headerLen, end: pos + size };
+}
+// A 64-bit field as a Number, clamped: nothing in a real file comes near 2^53, and all ones (the
+// "unknown" marker) has to stay recognisable after the conversion.
+const u64 = (buf, off) => { const v = buf.readBigUInt64BE(off); return v > BigInt(Number.MAX_SAFE_INTEGER) ? Number.MAX_SAFE_INTEGER : Number(v); };
+// A duration field that says "unknown": zero (fragmented files) or all ones.
+const unknownDur = (d, version) => d === 0 || d === (version === 1 ? Number.MAX_SAFE_INTEGER : 0xffffffff);
+
+/**
+ * The brand and length of an MP4/MOV, read from box headers with positioned reads — the moov
+ * may sit at the very end (a file written without "fast start"), and the file is never loaded.
+ * Returns { brand, durationSec } — durationSec null when the file does not say — or null for
+ * anything that is not a well-formed file: no ftyp first, no moov, no mvhd, a zero timescale, a
+ * box that overruns its parent, more than 10k boxes, or a moov over 8 MB.
+ */
+export function mp4Info(fd, size) {
+ try {
+ let pos = 0, count = 0, brand = null, moov = null;
+ while (pos < size) {
+ if (++count > MAX_BOXES) return null;
+ const b = boxAt(fd, pos, size);
+ if (!b) return null;
+ if (count === 1) {
+ if (b.type !== 'ftyp' || b.bodyLen < 4) return null;
+ brand = readAt(fd, b.body, 4).toString('latin1');
+ }
+ if (b.type === 'moov') { moov = b; break; }
+ pos = b.end;
+ }
+ if (!moov || moov.bodyLen + 8 > MOOV_MAX) return null;
+ let timescale = 0, dur = null, fragDur = null, sawMvhd = false;
+ pos = moov.body;
+ while (pos < moov.end) {
+ if (++count > MAX_BOXES) return null;
+ const b = boxAt(fd, pos, moov.end);
+ if (!b) return null;
+ if (b.type === 'mvhd' && !sawMvhd) {
+ sawMvhd = true;
+ const v = readAt(fd, b.body, Math.min(b.bodyLen, 32));
+ if (!v || v.length < 20) return null;
+ if (v[0] === 1) {
+ if (v.length < 32) return null;
+ timescale = v.readUInt32BE(20);
+ dur = u64(v, 24);
+ if (unknownDur(dur, 1)) dur = null;
+ } else {
+ timescale = v.readUInt32BE(12);
+ dur = v.readUInt32BE(16);
+ if (unknownDur(dur, 0)) dur = null;
+ }
+ } else if (b.type === 'mvex') {
+ // A fragmented file (Safari's MediaRecorder, some editors) leaves mvhd at 0 and states
+ // the whole length here, in mehd, in the same timescale.
+ let p = b.body;
+ while (p < b.end) {
+ if (++count > MAX_BOXES) return null;
+ const c = boxAt(fd, p, b.end);
+ if (!c) return null;
+ if (c.type === 'mehd') {
+ const v = readAt(fd, c.body, Math.min(c.bodyLen, 12));
+ if (v && v.length >= 8) {
+ const wide = v[0] === 1 && v.length >= 12;
+ const d = wide ? u64(v, 4) : v.readUInt32BE(4);
+ if (!unknownDur(d, wide ? 1 : 0)) fragDur = d;
+ }
+ }
+ p = c.end;
+ }
+ }
+ pos = b.end;
+ }
+ if (!sawMvhd || !timescale) return null;
+ const ticks = dur ?? fragDur;
+ return { brand, durationSec: ticks == null ? null : Math.round((ticks / timescale) * 1000) / 1000 };
+ } catch {
+ return null;
+ }
+}
+
+/* ---------------------------------------------------------------- references */
+
+/**
+ * Every blob a state refers to: customEx[].media.hash and customEx[].media.poster.hash.
+ * Deliberately loose — each field counts on its own as long as it is a well-formed hash, even
+ * when the rest of the ref would not pass the client's normalizeMediaRef. This set decides what
+ * is KEPT, so erring towards more is the safe direction. frontend/src/lib/media-refs.js has to
+ * agree with it; api/test/fixtures/media-refs.json pins both to the same answers.
+ */
+export function referencedHashes(state) {
+ const out = new Set();
+ const list = state && typeof state === 'object' && Array.isArray(state.customEx) ? state.customEx : [];
+ const add = r => { if (r && typeof r === 'object' && typeof r.hash === 'string' && HASH_RE.test(r.hash)) out.add(r.hash); };
+ for (const c of list) {
+ const m = c && typeof c === 'object' ? c.media : null;
+ if (!m || typeof m !== 'object') continue;
+ add(m);
+ add(m.poster);
+ }
+ return out;
+}
+
+/* ---------------------------------------------------------------- the store */
+
+const round1 = n => Math.round(n * 10) / 10;
+const contentType = req => String(req.headers?.['content-type'] || '').split(';')[0].trim().toLowerCase();
+function contentLength(req) {
+ const raw = req.headers?.['content-length'];
+ if (raw == null || raw === '') return null;
+ const n = Number(raw);
+ return Number.isSafeInteger(n) && n >= 0 ? n : null;
+}
+// Reads and discards what is left of a refused upload, so the answer reaches the client instead
+// of a connection reset (closing a socket with unread bytes makes the kernel send RST, and a
+// client that sees it before parsing the answer reports a dropped connection — see readBody in
+// server.js). A client that keeps sending past `limit` is not a mistaken one and is cut off.
+function drain(req, limit) {
+ if (req.readableEnded || req.destroyed) return;
+ let seen = 0;
+ req.on('data', d => { seen += d.length; if (seen > limit) req.destroy(); });
+ req.on('error', () => {});
+ req.resume();
+}
+
+/**
+ * Streams `req` into a new file at `file`, hashing and counting on the way. Resolves with
+ * { size, sha, head } once every byte is on disk and the file is closed. Rejects with a
+ * MediaError past `max` bytes (413) or after `idleMs` without a byte (408), with the socket's
+ * own error (marked clientGone) when the client hangs up, and with the disk's error otherwise.
+ * On every rejection the write stream has closed before the promise settles, so the caller can
+ * unlink the file without racing a late open().
+ */
+function streamToFile(req, file, { max, maxMB, idleMs }) {
+ return new Promise((resolve, reject) => {
+ const hash = crypto.createHash('sha256');
+ const head = [];
+ let headLen = 0, size = 0, ended = false, settled = false, timer = null;
+ const out = fs.createWriteStream(file, { flags: 'wx', mode: 0o600, highWaterMark: 256 * 1024 });
+ const arm = () => {
+ clearTimeout(timer);
+ timer = setTimeout(() => settle(new MediaError(408, 'timeout', {}, { Connection: 'close' })), idleMs);
+ };
+ const detach = () => {
+ clearTimeout(timer);
+ req.off('data', onData); req.off('end', onEnd); req.off('error', onReqError); req.off('close', onClose);
+ out.off('drain', onDrain);
+ };
+ function settle(err, val) {
+ if (settled) return;
+ settled = true;
+ detach();
+ if (!err) return resolve(val);
+ if (err.code === 'ENOSPC') err = new MediaError(507, 'storage-full');
+ if (out.closed) return reject(err);
+ out.once('close', () => reject(err));
+ out.destroy();
+ }
+ function onData(chunk) {
+ arm();
+ size += chunk.length;
+ if (size > max) return settle(new MediaError(413, 'media-too-large', { maxMB }));
+ hash.update(chunk);
+ if (headLen < HEAD_BYTES) {
+ // Copied, so the head does not keep a whole 64 KB socket chunk alive.
+ const part = Buffer.from(chunk.subarray(0, HEAD_BYTES - headLen));
+ head.push(part);
+ headLen += part.length;
+ }
+ if (!out.write(chunk)) req.pause();
+ }
+ function onDrain() { if (!settled && !ended) { arm(); req.resume(); } }
+ function onEnd() {
+ ended = true;
+ clearTimeout(timer);
+ out.end();
+ out.once('close', () => {
+ if (settled) return;
+ settled = true;
+ detach();
+ resolve({ size, sha: hash.digest('hex'), head: Buffer.concat(head) });
+ });
+ }
+ function onReqError(e) { settle(Object.assign(e || new Error('aborted'), { clientGone: true })); }
+ // 'close' before 'end' is a client that went away without an error event (an aborted
+ // request on some Node versions); after 'end' it is the ordinary end of the request.
+ function onClose() { if (!ended) settle(Object.assign(new Error('client went away mid-upload'), { clientGone: true })); }
+ out.on('error', e => settle(e));
+ out.on('drain', onDrain);
+ req.on('data', onData);
+ req.on('end', onEnd);
+ req.on('error', onReqError);
+ req.on('close', onClose);
+ arm();
+ req.resume();
+ });
+}
+
+/**
+ * The per-instance media store. `dir` is DATA_DIR/uploads. `readState(uid)` returns the
+ * profile's stored state, or null when there is none or it does not parse — the store sweeps
+ * nobody it gets null for. `now` is injectable so the tests can walk the clock.
+ */
+export function createMediaStore({ dir, limits, now = Date.now, readState = () => null, idleMs = 60000, log = console } = {}) {
+ const L = { ...mediaLimits({}), ...(limits || {}) };
+ const quotaBytes = L.quotaMB > 0 ? Math.round(L.quotaMB * MB) : 0;
+ const capMB = kind => (kind === 'video' ? L.videoMB : kind === 'gif' ? L.gifMB : L.imageMB);
+ const capOf = kind => Math.round(capMB(kind) * MB);
+ const maxCap = Math.max(capOf('image'), capOf('gif'), capOf('video'));
+ const users = new Map(); // safe uid -> { dir, bytes, count, reserved, inflight, hashes }
+ const orphansLogged = new Set();
+
+ // The same sanitising as server.js stateFile(). An id that sanitises to nothing would name
+ // uploads/ itself, and removeUser() would then delete everybody's files.
+ function safe(uid) {
+ const s = String(uid ?? '').replace(/[^a-zA-Z0-9_-]/g, '');
+ if (!s) throw new Error('media: a profile id is required');
+ return s;
+ }
+ function ensureDir(p) {
+ fs.mkdirSync(p, { recursive: true, mode: 0o700 });
+ // mkdir's mode is filtered by the umask and does nothing for a directory that already
+ // existed; the Coach runtime's uid must not be able to list anyone's files.
+ try { fs.chmodSync(p, 0o700); } catch { /* a host filesystem that refuses chmod */ }
+ }
+
+ // Built once per profile from one readdir, then kept current by every write and delete, so
+ // a state push never lists a directory.
+ function entry(uid) {
+ const id = safe(uid);
+ let e = users.get(id);
+ if (e) return e;
+ e = { id, dir: path.join(dir, id), bytes: 0, count: 0, reserved: 0, inflight: 0, hashes: new Map() };
+ let names = [];
+ try { names = fs.readdirSync(e.dir, { withFileTypes: true }); } catch { /* nothing uploaded yet */ }
+ for (const d of names) {
+ const m = FILE_RE.exec(d.name);
+ if (!m || !d.isFile() || e.hashes.has(m[1])) continue;
+ try {
+ const size = fs.statSync(path.join(e.dir, d.name)).size;
+ e.hashes.set(m[1], { ext: m[2], size });
+ e.bytes += size;
+ e.count++;
+ } catch { /* went away between readdir and stat */ }
+ }
+ users.set(id, e);
+ return e;
+ }
+ function forget(e, hash) {
+ const f = e.hashes.get(hash);
+ if (!f) return;
+ e.hashes.delete(hash);
+ e.bytes -= f.size;
+ e.count--;
+ }
+ const usageOf = e => ({ bytes: e.bytes, count: e.count, quotaBytes });
+
+ /* .gc.json: when each stored hash was last seen leaving the state. Read fresh every time (it
+ is a few hundred bytes), so what is on disk is the only truth. Unreadable = no marks: the
+ grace starts over, which can only ever keep a file longer. */
+ const gcFile = e => path.join(e.dir, '.gc.json');
+ function readMarks(e) {
+ try {
+ const raw = JSON.parse(fs.readFileSync(gcFile(e), 'utf8'));
+ if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return {};
+ const out = {};
+ for (const [h, t] of Object.entries(raw)) if (HASH_RE.test(h) && Number.isFinite(t)) out[h] = t;
+ return out;
+ } catch { return {}; }
+ }
+ function writeMarks(e, marks) {
+ const f = gcFile(e), tmp = f + '.tmp';
+ try {
+ fs.writeFileSync(tmp, JSON.stringify(marks), { mode: 0o600 });
+ fs.renameSync(tmp, f);
+ } catch (err) {
+ try { fs.unlinkSync(tmp); } catch { /* never written */ }
+ throw err;
+ }
+ }
+ // Brings the marks in line with `refs`: a referenced hash loses its mark, an unreferenced one
+ // without a mark gets `t`, marks for files that are gone are dropped. Returns whether anything
+ // changed.
+ function reconcile(e, refs, marks, t) {
+ let changed = false;
+ for (const h of e.hashes.keys()) {
+ if (refs.has(h)) { if (h in marks) { delete marks[h]; changed = true; } }
+ else if (!(h in marks)) { marks[h] = t; changed = true; }
+ }
+ for (const h of Object.keys(marks)) if (!e.hashes.has(h)) { delete marks[h]; changed = true; }
+ return changed;
+ }
+ function stateOf(uid) {
+ let S = null;
+ try { S = readState(uid); } catch { S = null; }
+ return S && typeof S === 'object' && !Array.isArray(S) ? S : null;
+ }
+
+ function noteState(uid, state) {
+ const id = safe(uid);
+ // Nothing to do for a profile that never uploaded — and that is the common case, so it
+ // costs one stat per profile per process, not a readdir per push.
+ if (!users.has(id) && !fs.existsSync(path.join(dir, id))) return false;
+ const e = entry(uid);
+ if (!e.hashes.size) return false;
+ const marks = readMarks(e);
+ const changed = reconcile(e, referencedHashes(state), marks, now());
+ if (changed) writeMarks(e, marks);
+ return changed;
+ }
+
+ function sweep(uid, { graceMs = L.gcGraceDays * DAY } = {}) {
+ const e = entry(uid);
+ const res = { removed: 0, freedBytes: 0, skipped: false };
+ if (!e.hashes.size) return res;
+ const S = stateOf(uid);
+ if (!S) { res.skipped = true; return res; } // never infer anything from a missing state
+ const refs = referencedHashes(S);
+ const marks = readMarks(e);
+ const t = now();
+ let changed = reconcile(e, refs, marks, t);
+ for (const [h, f] of [...e.hashes]) {
+ if (refs.has(h)) continue;
+ if (graceMs > 0 && !(t - marks[h] >= graceMs)) continue;
+ try { fs.unlinkSync(path.join(e.dir, `${h}.${f.ext}`)); }
+ catch (err) { if (err.code !== 'ENOENT') { log.error('media: could not delete', e.id, h, err.message); continue; } }
+ forget(e, h);
+ delete marks[h];
+ changed = true;
+ res.removed++;
+ res.freedBytes += f.size;
+ }
+ if (changed) {
+ try { writeMarks(e, marks); } catch (err) { log.error('media: could not write .gc.json for', e.id, err.message); }
+ }
+ return res;
+ }
+
+ // Temp files of uploads that died with the process, or that a crash between write and rename
+ // left behind. `olderThan` 0 = all of them (boot, when nothing can be in flight).
+ function cleanTmpOf(userDir, olderThan) {
+ const t = now();
+ const tmpDir = path.join(userDir, '.tmp');
+ let names = [];
+ try { names = fs.readdirSync(tmpDir); } catch { /* none */ }
+ let n = 0;
+ for (const name of names) {
+ const p = path.join(tmpDir, name);
+ try {
+ if (olderThan > 0 && t - fs.statSync(p).mtimeMs < olderThan) continue;
+ fs.rmSync(p, { recursive: true, force: true });
+ n++;
+ } catch { /* gone already */ }
+ }
+ try {
+ const g = path.join(userDir, '.gc.json.tmp');
+ if (olderThan === 0 || t - fs.statSync(g).mtimeMs >= olderThan) fs.unlinkSync(g);
+ } catch { /* none */ }
+ return n;
+ }
+ const userDirs = () => {
+ try { return fs.readdirSync(dir, { withFileTypes: true }).filter(d => d.isDirectory() && !d.name.startsWith('.')).map(d => d.name); }
+ catch { return []; }
+ };
+
+ function markIfUnreferenced(uid, e, hash) {
+ const S = stateOf(uid);
+ if (S && referencedHashes(S).has(hash)) return;
+ const marks = readMarks(e);
+ marks[hash] = now();
+ writeMarks(e, marks);
+ }
+ // A device just vouched for a file the server already has. If it was on its way out, the
+ // grace starts over: the state push that references it is usually a second behind.
+ function touch(e, hash) {
+ try {
+ const marks = readMarks(e);
+ if (!(hash in marks)) return;
+ marks[hash] = now();
+ writeMarks(e, marks);
+ } catch (err) { log.error('media: could not write .gc.json for', e.id, err.message); }
+ }
+ function freeBytes(p) {
+ try { const s = fs.statfsSync(p); return Number(s.bavail) * Number(s.bsize); }
+ catch { return null; } // a filesystem that cannot say: no floor rather than no uploads
+ }
+ const existed = (hash, f) => ({ status: 200, body: { ok: true, hash, mime: EXT_MIME[f.ext], size: f.size, existed: true } });
+
+ async function receive(uid, hash, req) {
+ if (!HASH_RE.test(String(hash))) { drain(req, maxCap); throw new MediaError(400, 'bad-request'); }
+ const e = entry(uid);
+ if (e.inflight >= MAX_INFLIGHT) {
+ drain(req, 2 * maxCap);
+ throw new MediaError(429, 'busy', { retryAfter: 5 }, { 'Retry-After': '5' });
+ }
+ e.inflight++;
+ let reserved = 0, tmp = null, cap = maxCap;
+ try {
+ const declared = MEDIA_TYPES[contentType(req)];
+ if (!declared) throw new MediaError(415, 'media-type');
+ cap = capOf(declared.kind);
+ const len = contentLength(req);
+ if (len != null && len > cap) throw new MediaError(413, 'media-too-large', { maxMB: capMB(declared.kind) });
+ const have = e.hashes.get(hash);
+ if (have) { drain(req, 2 * cap); touch(e, hash); return existed(hash, have); }
+
+ // Reserved before a byte is read, so two uploads at once cannot both fit into the space
+ // that is only left for one.
+ const need = len ?? cap;
+ if (quotaBytes && e.bytes + e.reserved + need > quotaBytes) {
+ // Space the person already let go of may still be sitting out its grace period; under
+ // pressure it gets an hour instead of two weeks.
+ try { sweep(uid, { graceMs: HOUR }); } catch (err) { log.error('media: quota sweep failed for', e.id, err.message); }
+ if (e.bytes + e.reserved + need > quotaBytes) {
+ throw new MediaError(413, 'media-quota', { usedMB: round1(e.bytes / MB), quotaMB: L.quotaMB });
+ }
+ }
+ e.reserved += need;
+ reserved = need;
+
+ ensureDir(dir);
+ ensureDir(e.dir);
+ ensureDir(path.join(e.dir, '.tmp'));
+ if (L.minFreeMB > 0) {
+ const free = freeBytes(e.dir);
+ if (free != null && free - need < L.minFreeMB * MB) throw new MediaError(507, 'storage-full');
+ }
+
+ tmp = path.join(e.dir, '.tmp', crypto.randomBytes(12).toString('hex'));
+ const got = await streamToFile(req, tmp, { max: cap, maxMB: capMB(declared.kind), idleMs });
+
+ if (got.sha !== hash) throw new MediaError(400, 'hash-mismatch');
+ const sn = sniffMedia(got.head);
+ if (!sn || sn.category !== declared.category) throw new MediaError(415, 'media-type');
+ if (got.size > capOf(sn.kind)) throw new MediaError(413, 'media-too-large', { maxMB: capMB(sn.kind) });
+ if (sn.ext === 'mp4' || sn.ext === 'mov') {
+ const fd = fs.openSync(tmp, 'r');
+ let info;
+ try { info = mp4Info(fd, got.size); } finally { fs.closeSync(fd); }
+ if (!info) throw new MediaError(415, 'media-invalid');
+ // One second of slack: the client measures with the platform's player, which rounds.
+ if (info.durationSec != null && info.durationSec > L.videoSec + 1) throw new MediaError(413, 'media-too-long', { maxSec: L.videoSec });
+ }
+
+ // The same bytes from another device may have landed while these were streaming. The
+ // rename and the cache update below are synchronous, so this check cannot be overtaken.
+ const now2 = e.hashes.get(hash);
+ if (now2) { touch(e, hash); return existed(hash, now2); }
+ fs.renameSync(tmp, path.join(e.dir, `${hash}.${sn.ext}`));
+ tmp = null;
+ e.hashes.set(hash, { ext: sn.ext, size: got.size });
+ e.bytes += got.size;
+ e.count++;
+ // Unreferenced until the state push that names it arrives; the mark is what lets the
+ // grace protect it until then, and what lets it go if that push never comes.
+ try { markIfUnreferenced(uid, e, hash); } catch (err) { log.error('media: could not mark', e.id, hash, err.message); }
+ return { status: 201, body: { ok: true, hash, mime: sn.mime, size: got.size, existed: false } };
+ } catch (err) {
+ if (err instanceof MediaError) drain(req, 2 * cap);
+ throw err;
+ } finally {
+ if (tmp) { try { fs.unlinkSync(tmp); } catch { /* never created, or already gone */ } }
+ e.reserved -= reserved;
+ e.inflight--;
+ }
+ }
+
+ return {
+ limits: L,
+ usage: uid => usageOf(entry(uid)),
+ has: (uid, hash) => entry(uid).hashes.has(hash),
+ /** { path, ext, mime, size } of a stored file, or null. Checks the disk, so a file that
+ * vanished from under the cache is forgotten rather than answered with a 500. */
+ file(uid, hash) {
+ const e = entry(uid);
+ const f = e.hashes.get(hash);
+ if (!f) return null;
+ const p = path.join(e.dir, `${hash}.${f.ext}`);
+ let st;
+ try { st = fs.statSync(p); } catch { forget(e, hash); return null; }
+ if (!st.isFile()) return null;
+ if (st.size !== f.size) { e.bytes += st.size - f.size; f.size = st.size; }
+ return { path: p, ext: f.ext, mime: EXT_MIME[f.ext], size: st.size };
+ },
+ receive,
+ missing(uid, hashes) {
+ const e = entry(uid);
+ return { missing: [...new Set(hashes)].filter(h => !e.hashes.has(h)), usage: usageOf(e) };
+ },
+ noteState,
+ sweep,
+ /** The hourly pass. Only profiles in `uids` are swept, and only when their state reads;
+ * a folder that belongs to nobody in db.json is left alone and said so, once per process —
+ * only an admin deleting the profile removes a folder. */
+ sweepAll({ uids = [], graceMs } = {}) {
+ const known = new Map();
+ for (const u of uids) { try { known.set(safe(u), u); } catch { /* unusable id */ } }
+ const out = { swept: 0, removed: 0, freedBytes: 0, skipped: 0, orphans: 0, tmp: 0 };
+ for (const name of userDirs()) {
+ out.tmp += cleanTmpOf(path.join(dir, name), HOUR);
+ const uid = known.get(name);
+ if (uid === undefined) {
+ out.orphans++;
+ if (!orphansLogged.has(name)) {
+ orphansLogged.add(name);
+ log.warn(`media: uploads/${name} belongs to no profile in db.json — left alone`);
+ }
+ continue;
+ }
+ try {
+ const r = sweep(uid, graceMs === undefined ? {} : { graceMs });
+ if (r.skipped) out.skipped++; else out.swept++;
+ out.removed += r.removed;
+ out.freedBytes += r.freedBytes;
+ } catch (err) { log.error('media: sweep failed for', name, err.message); }
+ }
+ return out;
+ },
+ /** Admin delete: the profile's whole folder. */
+ removeUser(uid) {
+ const id = safe(uid);
+ users.delete(id);
+ fs.rmSync(path.join(dir, id), { recursive: true, force: true });
+ },
+ /** Boot: nothing is in flight yet, so every temp file is a leftover. */
+ cleanTmp() {
+ let n = 0;
+ for (const name of userDirs()) n += cleanTmpOf(path.join(dir, name), 0);
+ return n;
+ }
+ };
+}
diff --git a/api/test/fixtures/media-refs.json b/api/test/fixtures/media-refs.json
new file mode 100644
index 000000000..f1b673b36
--- /dev/null
+++ b/api/test/fixtures/media-refs.json
@@ -0,0 +1,430 @@
+{
+ "about": "Which blob hashes a state references. api/media.js referencedHashes() and frontend/src/lib/media-refs.js referencedHashes() must both give `expect` (compared as sorted lists) for every `state` here: the server keeps and the client keeps exactly this set. Each of customEx[].media.hash and customEx[].media.poster.hash counts on its own when it matches /^[0-9a-f]{64}$/ — deliberately looser than normalizeMediaRef, because this set decides what is kept.",
+ "referencedHashes": [
+ {
+ "name": "no state at all",
+ "state": null,
+ "expect": []
+ },
+ {
+ "name": "a state without customEx",
+ "state": {
+ "unit": "kg"
+ },
+ "expect": []
+ },
+ {
+ "name": "customEx that is not a list",
+ "state": {
+ "customEx": {
+ "x": {
+ "id": "cx1",
+ "n": "Sandbag carry cx1",
+ "bp": "back",
+ "custom": true,
+ "media": {
+ "kind": "image",
+ "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "mime": "image/webp",
+ "size": 1000,
+ "width": 800,
+ "height": 600,
+ "at": 1
+ }
+ }
+ }
+ },
+ "expect": []
+ },
+ {
+ "name": "entries that are not exercises are skipped",
+ "state": {
+ "customEx": [
+ null,
+ 3,
+ "cx",
+ [],
+ {
+ "id": "cx1",
+ "n": "Sandbag carry cx1",
+ "bp": "back",
+ "custom": true,
+ "media": {
+ "kind": "image",
+ "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "mime": "image/webp",
+ "size": 1000,
+ "width": 800,
+ "height": 600,
+ "at": 1
+ }
+ }
+ ]
+ },
+ "expect": [
+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ ]
+ },
+ {
+ "name": "an exercise without media",
+ "state": {
+ "customEx": [
+ {
+ "id": "cx1",
+ "n": "Sandbag carry cx1",
+ "bp": "back",
+ "custom": true
+ }
+ ]
+ },
+ "expect": []
+ },
+ {
+ "name": "a photo and its poster",
+ "state": {
+ "customEx": [
+ {
+ "id": "cx1",
+ "n": "Sandbag carry cx1",
+ "bp": "back",
+ "custom": true,
+ "media": {
+ "kind": "image",
+ "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "mime": "image/webp",
+ "size": 1000,
+ "width": 800,
+ "height": 600,
+ "poster": {
+ "hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+ "mime": "image/webp",
+ "size": 200,
+ "width": 480,
+ "height": 360
+ },
+ "at": 1
+ }
+ }
+ ]
+ },
+ "expect": [
+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+ ]
+ },
+ {
+ "name": "a video without a poster",
+ "state": {
+ "customEx": [
+ {
+ "id": "cx1",
+ "n": "Sandbag carry cx1",
+ "bp": "back",
+ "custom": true,
+ "media": {
+ "kind": "video",
+ "hash": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
+ "mime": "video/mp4",
+ "size": 5000000,
+ "width": 1920,
+ "height": 1080,
+ "dur": 12.4,
+ "codec": "avc1",
+ "at": 1
+ }
+ }
+ ]
+ },
+ "expect": [
+ "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
+ ]
+ },
+ {
+ "name": "the same file on two exercises counts once",
+ "state": {
+ "customEx": [
+ {
+ "id": "cx1",
+ "n": "Sandbag carry cx1",
+ "bp": "back",
+ "custom": true,
+ "media": {
+ "kind": "image",
+ "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "mime": "image/webp",
+ "size": 1000,
+ "width": 800,
+ "height": 600,
+ "poster": {
+ "hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+ "mime": "image/webp",
+ "size": 200,
+ "width": 480,
+ "height": 360
+ },
+ "at": 1
+ }
+ },
+ {
+ "id": "cx2",
+ "n": "Sandbag carry cx2",
+ "bp": "back",
+ "custom": true,
+ "media": {
+ "kind": "image",
+ "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "mime": "image/webp",
+ "size": 1000,
+ "width": 800,
+ "height": 600,
+ "poster": {
+ "hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+ "mime": "image/webp",
+ "size": 200,
+ "width": 480,
+ "height": 360
+ },
+ "at": 1
+ }
+ }
+ ]
+ },
+ "expect": [
+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+ ]
+ },
+ {
+ "name": "an upper-case hash is not a hash",
+ "state": {
+ "customEx": [
+ {
+ "id": "cx1",
+ "n": "Sandbag carry cx1",
+ "bp": "back",
+ "custom": true,
+ "media": {
+ "kind": "image",
+ "hash": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+ "mime": "image/webp",
+ "size": 1000,
+ "width": 800,
+ "height": 600,
+ "at": 1
+ }
+ }
+ ]
+ },
+ "expect": []
+ },
+ {
+ "name": "a hash one character short is not a hash",
+ "state": {
+ "customEx": [
+ {
+ "id": "cx1",
+ "n": "Sandbag carry cx1",
+ "bp": "back",
+ "custom": true,
+ "media": {
+ "kind": "image",
+ "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "mime": "image/webp",
+ "size": 1000,
+ "width": 800,
+ "height": 600,
+ "at": 1
+ }
+ }
+ ]
+ },
+ "expect": []
+ },
+ {
+ "name": "a hash with a path in it is not a hash",
+ "state": {
+ "customEx": [
+ {
+ "id": "cx1",
+ "n": "Sandbag carry cx1",
+ "bp": "back",
+ "custom": true,
+ "media": {
+ "kind": "image",
+ "hash": "../aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "mime": "image/webp",
+ "size": 1000,
+ "width": 800,
+ "height": 600,
+ "at": 1
+ }
+ }
+ ]
+ },
+ "expect": []
+ },
+ {
+ "name": "a hash that is not a string",
+ "state": {
+ "customEx": [
+ {
+ "id": "cx1",
+ "n": "Sandbag carry cx1",
+ "bp": "back",
+ "custom": true,
+ "media": {
+ "kind": "image",
+ "hash": 12345,
+ "mime": "image/webp",
+ "size": 1000,
+ "width": 800,
+ "height": 600,
+ "at": 1
+ }
+ }
+ ]
+ },
+ "expect": []
+ },
+ {
+ "name": "media that is not an object",
+ "state": {
+ "customEx": [
+ {
+ "id": "cx1",
+ "n": "Sandbag carry cx1",
+ "bp": "back",
+ "custom": true,
+ "media": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ },
+ {
+ "id": "cx2",
+ "n": "Sandbag carry cx2",
+ "bp": "back",
+ "custom": true,
+ "media": [
+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ ]
+ }
+ ]
+ },
+ "expect": []
+ },
+ {
+ "name": "each field counts on its own: a good poster under a malformed main hash is kept",
+ "state": {
+ "customEx": [
+ {
+ "id": "cx1",
+ "n": "Sandbag carry cx1",
+ "bp": "back",
+ "custom": true,
+ "media": {
+ "kind": "image",
+ "hash": "nope",
+ "mime": "image/webp",
+ "size": 1000,
+ "width": 800,
+ "height": 600,
+ "poster": {
+ "hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+ "mime": "image/webp",
+ "size": 200,
+ "width": 480,
+ "height": 360
+ },
+ "at": 1
+ }
+ }
+ ]
+ },
+ "expect": [
+ "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+ ]
+ },
+ {
+ "name": "a ref the client would not render (unknown mime) still keeps its file",
+ "state": {
+ "customEx": [
+ {
+ "id": "cx1",
+ "n": "Sandbag carry cx1",
+ "bp": "back",
+ "custom": true,
+ "media": {
+ "kind": "image",
+ "hash": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+ "mime": "image/svg+xml",
+ "size": 1000,
+ "width": 800,
+ "height": 600,
+ "at": 1
+ }
+ }
+ ]
+ },
+ "expect": [
+ "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+ ]
+ },
+ {
+ "name": "stray img/gif fields from a fork build are not references",
+ "state": {
+ "customEx": [
+ {
+ "id": "cx1",
+ "n": "Sandbag carry cx1",
+ "bp": "back",
+ "custom": true,
+ "img": "http://x/y.jpg",
+ "gif": "data:image/gif;base64,R0lG"
+ }
+ ]
+ },
+ "expect": []
+ },
+ {
+ "name": "only customEx is a source: media-like fields elsewhere do not count",
+ "state": {
+ "routines": [
+ {
+ "id": "r1",
+ "ex": [
+ {
+ "id": "cx1",
+ "media": {
+ "kind": "image",
+ "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "mime": "image/webp",
+ "size": 1000,
+ "width": 800,
+ "height": 600,
+ "at": 1
+ }
+ }
+ ]
+ }
+ ],
+ "workouts": [
+ {
+ "entries": [
+ {
+ "media": {
+ "kind": "image",
+ "hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+ "mime": "image/webp",
+ "size": 1000,
+ "width": 800,
+ "height": 600,
+ "at": 1
+ }
+ }
+ ]
+ }
+ ],
+ "customEx": []
+ },
+ "expect": []
+ }
+ ]
+}
diff --git a/api/test/media-gc.test.js b/api/test/media-gc.test.js
new file mode 100644
index 000000000..5e197eff9
--- /dev/null
+++ b/api/test/media-gc.test.js
@@ -0,0 +1,249 @@
+/* Which stored photos and videos go, and — the half that matters more — which never do.
+ A file goes when the profile's own readable state has not referenced it for the grace period
+ (14 days; one hour when the quota is full; none for an explicit sweep). Nothing is ever
+ deleted because something is missing or unreadable: the state file, the user in db.json, or
+ .gc.json. Other devices and stashes may still hold refs that are gone from the server state,
+ and the grace is what keeps their files for them. In-process, with the clock injected. */
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { createMediaStore, mediaLimits, MediaError } from '../media.js';
+import * as M from './media-samples.mjs';
+
+const DAY = 86400000, HOUR = 3600000;
+const quiet = { warn() {}, error() {}, log() {} };
+
+function setup(t, { limits = {}, states = {} } = {}) {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'media-gc-'));
+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+ const dir = path.join(root, 'uploads');
+ const clock = { t: 1_800_000_000_000 };
+ const S = new Map(Object.entries(states));
+ const warnings = [];
+ const store = createMediaStore({
+ dir, limits: { ...mediaLimits({}), ...limits }, now: () => clock.t,
+ readState: uid => {
+ const v = S.get(uid);
+ if (v instanceof Error) throw v;
+ return v ?? null;
+ },
+ log: { ...quiet, warn: m => warnings.push(m) }
+ });
+ // Puts a file in place the way an upload leaves it, before the store has looked.
+ const place = (uid, bytes, ext = 'jpg') => {
+ const d = path.join(dir, uid);
+ fs.mkdirSync(d, { recursive: true, mode: 0o700 });
+ const h = M.sha(bytes);
+ fs.writeFileSync(path.join(d, `${h}.${ext}`), bytes, { mode: 0o600 });
+ return h;
+ };
+ const files = uid => { try { return fs.readdirSync(path.join(dir, uid)).filter(n => !n.startsWith('.')).sort(); } catch { return []; } };
+ const marks = uid => JSON.parse(fs.readFileSync(path.join(dir, uid, '.gc.json'), 'utf8'));
+ return { dir, clock, S, store, place, files, marks, warnings };
+}
+const ref = (...hashes) => ({ customEx: hashes.map((h, i) => ({ id: 'cx' + i, n: 'x', bp: 'back', custom: true, media: { kind: 'image', hash: h, mime: 'image/jpeg', size: 1, width: 1, height: 1, at: 1 } })) });
+
+test('noteState marks a file the state stopped referencing and clears the mark when it is back', t => {
+ const h = setup(t);
+ const a = h.place('u1', M.jpeg()), b = h.place('u1', M.jpeg());
+ assert.equal(h.store.noteState('u1', ref(a, b)), false, 'both referenced: nothing to write');
+ assert.equal(fs.existsSync(path.join(h.dir, 'u1', '.gc.json')), false);
+
+ assert.equal(h.store.noteState('u1', ref(a)), true);
+ assert.deepEqual(h.marks('u1'), { [b]: h.clock.t });
+
+ h.clock.t += DAY;
+ assert.equal(h.store.noteState('u1', ref(a)), false, 'an existing mark keeps its time');
+ assert.deepEqual(h.marks('u1'), { [b]: h.clock.t - DAY });
+
+ assert.equal(h.store.noteState('u1', ref(a, b)), true);
+ assert.deepEqual(h.marks('u1'), {});
+});
+
+test('noteState does nothing at all for a profile that never uploaded', t => {
+ const h = setup(t);
+ assert.equal(h.store.noteState('u1', ref('a'.repeat(64))), false);
+ assert.equal(fs.existsSync(h.dir), false, 'no uploads folder is created by a state push');
+});
+
+test('the hourly sweep keeps referenced files and removes the rest only after the grace', t => {
+ const h = setup(t);
+ const keep = h.place('u1', M.jpeg()), drop = h.place('u1', M.jpeg(3000));
+ h.S.set('u1', ref(keep));
+ assert.deepEqual(h.store.usage('u1'), { bytes: 4000, count: 2, quotaBytes: 200 * 1048576 });
+ h.store.noteState('u1', ref(keep));
+
+ h.clock.t += 14 * DAY - 1;
+ assert.deepEqual(h.store.sweep('u1'), { removed: 0, freedBytes: 0, skipped: false });
+ assert.equal(h.files('u1').length, 2, 'a day short of the grace: still there');
+
+ h.clock.t += 1;
+ assert.deepEqual(h.store.sweep('u1'), { removed: 1, freedBytes: 3000, skipped: false });
+ assert.deepEqual(h.files('u1'), [`${keep}.jpg`]);
+ assert.deepEqual(h.store.usage('u1'), { bytes: 1000, count: 1, quotaBytes: 200 * 1048576 });
+ assert.deepEqual(h.marks('u1'), {}, 'the mark goes with the file');
+});
+
+test('a file nobody marked yet starts its grace at the first sweep that sees it unreferenced', t => {
+ const h = setup(t);
+ const x = h.place('u1', M.jpeg());
+ h.S.set('u1', ref());
+ assert.equal(h.store.sweep('u1').removed, 0);
+ assert.deepEqual(h.marks('u1'), { [x]: h.clock.t });
+ h.clock.t += 14 * DAY;
+ assert.equal(h.store.sweep('u1').removed, 1);
+});
+
+test('an explicit sweep (grace 0) removes every unreferenced file at once', t => {
+ const h = setup(t);
+ const keep = h.place('u1', M.jpeg()), a = h.place('u1', M.png()), b = h.place('u1', M.gif(), 'gif');
+ h.S.set('u1', ref(keep));
+ const r = h.store.sweep('u1', { graceMs: 0 });
+ assert.equal(r.removed, 2);
+ assert.deepEqual(h.files('u1'), [`${keep}.jpg`]);
+ assert.ok(!h.store.has('u1', a) && !h.store.has('u1', b));
+});
+
+test('a profile whose state cannot be read is never swept — missing, unparsable or throwing', t => {
+ const h = setup(t);
+ for (const [uid, state] of [['gone', undefined], ['bad', null], ['arr', []], ['boom', new Error('EIO')]]) {
+ h.place(uid, M.jpeg());
+ if (state !== undefined) h.S.set(uid, state);
+ h.clock.t += 30 * DAY;
+ const r = h.store.sweep(uid, { graceMs: 0 });
+ assert.deepEqual(r, { removed: 0, freedBytes: 0, skipped: true }, uid);
+ assert.equal(h.files(uid).length, 1, uid);
+ }
+});
+
+test('an unreadable .gc.json is no marks: the grace starts over and nothing is deleted for it', t => {
+ const h = setup(t);
+ const x = h.place('u1', M.jpeg());
+ h.S.set('u1', ref());
+ h.store.sweep('u1'); // marked now
+ h.clock.t += 20 * DAY;
+ fs.writeFileSync(path.join(h.dir, 'u1', '.gc.json'), '{"torn');
+ assert.equal(h.store.sweep('u1').removed, 0, 'a mark that cannot be read is not a mark');
+ assert.deepEqual(h.marks('u1'), { [x]: h.clock.t }, 'a fresh mark replaces the torn file');
+ h.clock.t += 14 * DAY;
+ assert.equal(h.store.sweep('u1').removed, 1);
+});
+
+test('.gc.json turned into a directory: the hourly sweep deletes nothing, noteState throws for the caller to catch', t => {
+ const h = setup(t);
+ h.place('u1', M.jpeg());
+ h.S.set('u1', ref());
+ fs.mkdirSync(path.join(h.dir, 'u1', '.gc.json'));
+ h.clock.t += 60 * DAY;
+ assert.equal(h.store.sweep('u1').removed, 0);
+ assert.equal(h.files('u1').length, 1);
+ assert.throws(() => h.store.noteState('u1', { customEx: [] }));
+ assert.equal(fs.existsSync(path.join(h.dir, 'u1', '.gc.json.tmp')), false, 'the failed write leaves no temp file');
+});
+
+test('sweepAll with an empty db.users removes no folder and no file, and says so once', t => {
+ const h = setup(t);
+ h.place('u1', M.jpeg());
+ h.place('u2', M.jpeg());
+ h.S.set('u1', ref());
+ h.S.set('u2', ref());
+ h.clock.t += 60 * DAY;
+ for (let i = 0; i < 3; i++) {
+ const r = h.store.sweepAll({ uids: [], graceMs: 0 });
+ assert.equal(r.removed, 0);
+ assert.equal(r.orphans, 2);
+ }
+ assert.equal(h.files('u1').length + h.files('u2').length, 2);
+ assert.equal(h.warnings.length, 2, 'one line per folder per process, not one per hour');
+ assert.match(h.warnings[0], /belongs to no profile/);
+});
+
+test('sweepAll sweeps the profiles it is given, and only when their state reads', t => {
+ const h = setup(t);
+ h.place('u1', M.jpeg());
+ h.place('u2', M.jpeg());
+ h.S.set('u1', ref()); // readable, references nothing
+ // u2: no state at all
+ const r = h.store.sweepAll({ uids: ['u1', 'u2'], graceMs: 0 });
+ assert.equal(r.removed, 1);
+ assert.equal(r.swept, 1);
+ assert.equal(r.skipped, 1);
+ assert.equal(h.files('u1').length, 0);
+ assert.equal(h.files('u2').length, 1);
+});
+
+test('temp files: the hourly pass removes those older than an hour, boot removes them all', t => {
+ const h = setup(t);
+ h.place('u1', M.jpeg());
+ const tmp = path.join(h.dir, 'u1', '.tmp');
+ fs.mkdirSync(tmp, { recursive: true });
+ fs.writeFileSync(path.join(tmp, 'old'), 'x');
+ fs.writeFileSync(path.join(tmp, 'new'), 'x');
+ const old = new Date(h.clock.t - 2 * HOUR), fresh = new Date(h.clock.t - 10 * 60000);
+ fs.utimesSync(path.join(tmp, 'old'), old, old);
+ fs.utimesSync(path.join(tmp, 'new'), fresh, fresh);
+ const r = h.store.sweepAll({ uids: [] });
+ assert.equal(r.tmp, 1);
+ assert.deepEqual(fs.readdirSync(tmp), ['new']);
+ assert.equal(h.store.cleanTmp(), 1);
+ assert.deepEqual(fs.readdirSync(tmp), []);
+});
+
+test('dotfiles and unknown names are never blobs', t => {
+ const h = setup(t);
+ const x = h.place('u1', M.jpeg());
+ const d = path.join(h.dir, 'u1');
+ fs.writeFileSync(path.join(d, '.gc.json'), '{}');
+ fs.writeFileSync(path.join(d, `${x}.svg`), '<svg/>');
+ fs.writeFileSync(path.join(d, 'notes.txt'), 'hello');
+ fs.writeFileSync(path.join(d, `.${'e'.repeat(64)}.jpg`), 'x');
+ assert.deepEqual(h.store.usage('u1'), { bytes: 1000, count: 1, quotaBytes: 200 * 1048576 });
+ h.S.set('u1', ref());
+ h.store.sweep('u1', { graceMs: 0 });
+ assert.deepEqual(fs.readdirSync(d).sort(), ['.gc.json', `.${'e'.repeat(64)}.jpg`, '.tmp', `${x}.svg`, 'notes.txt'].filter(n => n !== '.tmp').sort());
+});
+
+test('removeUser deletes the whole folder, and an id that sanitises to nothing is refused', t => {
+ const h = setup(t);
+ h.place('u1', M.jpeg());
+ h.place('u2', M.jpeg());
+ h.store.removeUser('u1');
+ assert.equal(fs.existsSync(path.join(h.dir, 'u1')), false);
+ assert.equal(fs.existsSync(path.join(h.dir, 'u2')), true);
+ assert.equal(h.store.usage('u1').count, 0);
+ // '../..' would otherwise name uploads/ itself — everybody's files.
+ assert.throws(() => h.store.removeUser('../..'));
+ assert.throws(() => h.store.removeUser(''));
+ assert.equal(fs.existsSync(path.join(h.dir, 'u2')), true);
+});
+
+test('under quota pressure the grace is one hour', async t => {
+ // 2500 bytes of quota; 1000 already stored and unreferenced; a 2000-byte upload needs room.
+ const h = setup(t, { limits: { quotaMB: 2500 / 1048576, minFreeMB: 0 } });
+ const old = h.place('u1', M.jpeg());
+ h.S.set('u1', ref());
+ h.store.noteState('u1', ref()); // marked now
+ const next = M.jpeg(2000);
+
+ h.clock.t += 30 * 60000; // half an hour: still protected
+ await assert.rejects(h.store.receive('u1', M.sha(next), M.fakeReq(next)),
+ e => e instanceof MediaError && e.status === 413 && e.code === 'media-quota' && e.extra.quotaMB > 0);
+ assert.ok(h.store.has('u1', old));
+
+ h.clock.t += 31 * 60000; // past the hour: it makes room
+ const r = await h.store.receive('u1', M.sha(next), M.fakeReq(next));
+ assert.equal(r.status, 201);
+ assert.ok(!h.store.has('u1', old));
+ assert.equal(h.store.usage('u1').bytes, 2000);
+});
+
+test('a new upload nobody references yet is marked, so the grace also covers a push that never comes', async t => {
+ const h = setup(t, { limits: { minFreeMB: 0 } });
+ const a = M.jpeg(), b = M.jpeg();
+ h.S.set('u1', ref(M.sha(b)));
+ await h.store.receive('u1', M.sha(a), M.fakeReq(a));
+ await h.store.receive('u1', M.sha(b), M.fakeReq(b));
+ assert.deepEqual(h.marks('u1'), { [M.sha(a)]: h.clock.t }, 'the referenced one is not marked');
+});
diff --git a/api/test/media-receive.test.js b/api/test/media-receive.test.js
new file mode 100644
index 000000000..fc29d3e7d
--- /dev/null
+++ b/api/test/media-receive.test.js
@@ -0,0 +1,207 @@
+/* receive(): one upload from the first byte to the rename. In-process, so the memory it holds
+ and the state it leaves behind on every way out can be looked at directly. The production API
+ runs with a 128 MB memory limit and a video may be 40 MB: the bytes must stream to disk, never
+ collect in memory. And whatever ends an upload — success, refusal, a client hanging up, a
+ stall — the temp file goes, and the in-flight slot and the quota reservation come back. */
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import v8 from 'node:v8';
+import vm from 'node:vm';
+import { Readable } from 'node:stream';
+import { createMediaStore, mediaLimits, MediaError } from '../media.js';
+import * as M from './media-samples.mjs';
+
+v8.setFlagsFromString('--expose-gc');
+const gc = vm.runInNewContext('gc');
+const quiet = { warn() {}, error() {}, log() {} };
+
+function setup(t, limits = {}, opts = {}) {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'media-recv-'));
+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+ const dir = path.join(root, 'uploads');
+ const store = createMediaStore({ dir, limits: { ...mediaLimits({}), minFreeMB: 0, ...limits }, log: quiet, ...opts });
+ const tmpFiles = uid => { try { return fs.readdirSync(path.join(dir, uid, '.tmp')); } catch { return []; } };
+ const files = uid => { try { return fs.readdirSync(path.join(dir, uid)).filter(n => !n.startsWith('.')); } catch { return []; } };
+ return { dir, store, tmpFiles, files };
+}
+const isMediaError = (status, code) => e => e instanceof MediaError && e.status === status && e.code === code;
+
+test('a 30 MB upload streams through with a flat memory profile', async t => {
+ const h = setup(t);
+ // A real MP4 head so the whole path runs, sniff and box walk included; the rest is fresh
+ // 64 KB buffers, each its own ArrayBuffer, so a receiver that kept them would show.
+ const headPart = M.mp4({ seconds: 20, mdatBytes: 0 });
+ const TOTAL = 30 * 1024 * 1024, CHUNK = 64 * 1024;
+ const bodyLen = TOTAL - headPart.length;
+ // mdat's size field has to cover what follows it for mp4Info to accept the file.
+ headPart.writeUInt32BE(8 + bodyLen, headPart.length - 8);
+ const crypto = await import('node:crypto');
+ const hasher = crypto.createHash('sha256').update(headPart);
+ const lenAt = i => Math.min(CHUNK, bodyLen - i * CHUNK);
+ const fillOf = i => (i * 7) & 0xff;
+ const chunkAt = i => Buffer.allocUnsafeSlow(lenAt(i)).fill(fillOf(i));
+ const nChunks = Math.ceil(bodyLen / CHUNK);
+ // Hashed through one scratch buffer: 30 MB of garbage made here would still be waiting for
+ // the collector when the baseline is taken, and its release would then hide real growth.
+ const scratch = Buffer.alloc(CHUNK);
+ for (let i = 0; i < nChunks; i++) hasher.update(scratch.fill(fillOf(i)).subarray(0, lenAt(i)));
+ const hash = hasher.digest('hex');
+
+ // ArrayBuffer backing stores are released by a task after the collection, not inside it.
+ const settle = async () => { gc(); await new Promise(r => setImmediate(r)); gc(); };
+ await settle();
+ const base = process.memoryUsage().arrayBuffers;
+ let peak = 0;
+ async function* body() {
+ yield headPart;
+ for (let i = 0; i < nChunks; i++) {
+ if (i % 32 === 0) { gc(); peak = Math.max(peak, process.memoryUsage().arrayBuffers - base); }
+ yield chunkAt(i);
+ }
+ }
+ const req = Readable.from(body(), { objectMode: false });
+ req.headers = { 'content-type': 'video/mp4', 'content-length': String(TOTAL) };
+ const r = await h.store.receive('u1', hash, req);
+ assert.equal(r.status, 201);
+ assert.equal(r.body.size, TOTAL);
+ await settle();
+ peak = Math.max(peak, process.memoryUsage().arrayBuffers - base);
+ assert.ok(peak < 8 * 1024 * 1024, `arrayBuffers grew by ${(peak / 1048576).toFixed(1)} MB`);
+ assert.equal(fs.statSync(path.join(h.dir, 'u1', `${hash}.mp4`)).size, TOTAL);
+});
+
+test('a client that hangs up mid-upload leaves no temp file and gives back its slot and reservation', async t => {
+ const h = setup(t, { quotaMB: 1 });
+ const bytes = M.jpeg(500000);
+ let sent = 0;
+ const req = new Readable({
+ read() {
+ if (sent >= 3) { this.destroy(Object.assign(new Error('aborted'), { code: 'ECONNRESET' })); return; }
+ this.push(bytes.subarray(sent * 65536, ++sent * 65536));
+ }
+ });
+ req.headers = { 'content-type': 'image/jpeg', 'content-length': String(bytes.length) };
+ await assert.rejects(h.store.receive('u1', M.sha(bytes), req), e => e.clientGone === true);
+ assert.deepEqual(h.tmpFiles('u1'), []);
+ assert.deepEqual(h.files('u1'), []);
+ // The reservation came back: a file that needs almost the whole quota fits.
+ const full = M.jpeg(1000000);
+ assert.equal((await h.store.receive('u1', M.sha(full), M.fakeReq(full))).status, 201);
+ // And the slot: two more at once are both taken.
+ const a = M.jpeg(1000), b = M.jpeg(1000);
+ const h2 = setup(t);
+ const results = await Promise.all([h2.store.receive('u1', M.sha(a), M.fakeReq(a)), h2.store.receive('u1', M.sha(b), M.fakeReq(b))]);
+ assert.deepEqual(results.map(r => r.status), [201, 201]);
+});
+
+test('an upload that stops sending is cut off after the idle time with 408', async t => {
+ const h = setup(t, {}, { idleMs: 80 });
+ const bytes = M.jpeg(200000);
+ const req = new Readable({ read() {} }); // pushes one chunk, then nothing, ever
+ req.push(bytes.subarray(0, 65536));
+ req.headers = { 'content-type': 'image/jpeg', 'content-length': String(bytes.length) };
+ await assert.rejects(h.store.receive('u1', M.sha(bytes), req), isMediaError(408, 'timeout'));
+ assert.deepEqual(h.tmpFiles('u1'), []);
+});
+
+test('an upload without a length is capped while streaming, not after', async t => {
+ const h = setup(t, { imageMB: 0.1 });
+ const bytes = M.jpeg(300000);
+ const req = M.fakeReq(bytes, { length: null });
+ await assert.rejects(h.store.receive('u1', M.sha(bytes), req), e => isMediaError(413, 'media-too-large')(e) && e.extra.maxMB === 0.1);
+ assert.deepEqual(h.tmpFiles('u1'), []);
+ assert.deepEqual(h.store.usage('u1'), { bytes: 0, count: 0, quotaBytes: 200 * 1048576 });
+});
+
+test('a declared length over the cap is refused before a byte is read', async t => {
+ const h = setup(t, { imageMB: 0.1 });
+ const bytes = M.jpeg(300000);
+ await assert.rejects(h.store.receive('u1', M.sha(bytes), M.fakeReq(bytes)), isMediaError(413, 'media-too-large'));
+ assert.equal(fs.existsSync(path.join(h.dir, 'u1', '.tmp')), false, 'no temp file was ever opened');
+});
+
+test('the file is stored as what it is: a JPEG declared as a GIF is a JPEG, over the image cap it is refused', async t => {
+ const h = setup(t, { imageMB: 0.01, gifMB: 1 });
+ const small = M.jpeg(5000);
+ const r = await h.store.receive('u1', M.sha(small), M.fakeReq(small, { declare: 'image/gif' }));
+ assert.deepEqual(r.body, { ok: true, hash: M.sha(small), mime: 'image/jpeg', size: 5000, existed: false });
+ assert.deepEqual(h.files('u1'), [`${M.sha(small)}.jpg`]);
+ const big = M.jpeg(50000);
+ await assert.rejects(h.store.receive('u1', M.sha(big), M.fakeReq(big, { declare: 'image/gif' })), e => isMediaError(413, 'media-too-large')(e) && e.extra.maxMB === 0.01);
+});
+
+test('a video declared as an image, SVG declared as PNG, and an unknown type are refused', async t => {
+ const h = setup(t);
+ const vid = M.mp4();
+ await assert.rejects(h.store.receive('u1', M.sha(vid), M.fakeReq(vid, { declare: 'image/jpeg' })), isMediaError(415, 'media-type'));
+ const svg = M.svg();
+ await assert.rejects(h.store.receive('u1', M.sha(svg), M.fakeReq(svg, { declare: 'image/png' })), isMediaError(415, 'media-type'));
+ await assert.rejects(h.store.receive('u1', M.sha(svg), M.fakeReq(svg, { declare: 'image/svg+xml' })), isMediaError(415, 'media-type'));
+ await assert.rejects(h.store.receive('u1', M.sha(svg), M.fakeReq(svg, { declare: 'text/html' })), isMediaError(415, 'media-type'));
+ assert.deepEqual(h.files('u1'), []);
+ assert.deepEqual(h.tmpFiles('u1'), []);
+});
+
+test('bytes that do not hash to their name are refused and leave nothing', async t => {
+ const h = setup(t);
+ const bytes = M.jpeg();
+ await assert.rejects(h.store.receive('u1', 'f'.repeat(64), M.fakeReq(bytes)), isMediaError(400, 'hash-mismatch'));
+ assert.deepEqual(h.files('u1'), []);
+ assert.deepEqual(h.tmpFiles('u1'), []);
+});
+
+test('videos: too long, unreadable, and a WebM that only the byte cap applies to', async t => {
+ const h = setup(t, { videoSec: 60 });
+ const ok = M.mp4({ seconds: 61 }); // one second of slack for rounding
+ assert.equal((await h.store.receive('u1', M.sha(ok), M.fakeReq(ok, { declare: 'video/mp4' }))).status, 201);
+ const long = M.mp4({ seconds: 62 });
+ await assert.rejects(h.store.receive('u1', M.sha(long), M.fakeReq(long, { declare: 'video/mp4' })), e => isMediaError(413, 'media-too-long')(e) && e.extra.maxSec === 60);
+ const broken = Buffer.concat([M.ftyp(), M.box('mdat', Buffer.alloc(100))]);
+ await assert.rejects(h.store.receive('u1', M.sha(broken), M.fakeReq(broken, { declare: 'video/mp4' })), isMediaError(415, 'media-invalid'));
+ const mov = M.mp4({ brand: 'qt ', seconds: 10 });
+ const r = await h.store.receive('u1', M.sha(mov), M.fakeReq(mov, { declare: 'video/mp4' }));
+ assert.equal(r.body.mime, 'video/quicktime');
+ const webm = M.ebml('webm', 5000);
+ assert.equal((await h.store.receive('u1', M.sha(webm), M.fakeReq(webm, { declare: 'video/webm' }))).body.mime, 'video/webm');
+ assert.deepEqual(h.files('u1').map(n => n.split('.')[1]).sort(), ['mov', 'mp4', 'webm']);
+});
+
+test('a file the server already has is answered as such and counted once', async t => {
+ const h = setup(t);
+ const bytes = M.png(4000);
+ const first = await h.store.receive('u1', M.sha(bytes), M.fakeReq(bytes, { declare: 'image/png' }));
+ const again = await h.store.receive('u1', M.sha(bytes), M.fakeReq(bytes, { declare: 'image/png' }));
+ assert.equal(first.status, 201);
+ assert.deepEqual([again.status, again.body], [200, { ok: true, hash: M.sha(bytes), mime: 'image/png', size: 4000, existed: true }]);
+ assert.deepEqual(h.store.usage('u1'), { bytes: 4000, count: 1, quotaBytes: 200 * 1048576 });
+ // The same bytes from two devices at once: one file, counted once.
+ const both = M.jpeg(300000);
+ const rs = await Promise.all([1, 2].map(() => h.store.receive('u2', M.sha(both), M.fakeReq(both, { chunk: 4096 }))));
+ assert.deepEqual(rs.map(r => r.status).sort(), [200, 201]);
+ assert.deepEqual(h.store.usage('u2'), { bytes: 300000, count: 1, quotaBytes: 200 * 1048576 });
+ assert.deepEqual(h.tmpFiles('u2'), []);
+});
+
+test('a third upload at once is refused as busy', async t => {
+ const h = setup(t);
+ const stalled = () => { const r = new Readable({ read() {} }); r.headers = { 'content-type': 'image/jpeg', 'content-length': '100000' }; return r; };
+ const a = stalled(), b = stalled();
+ const pa = h.store.receive('u1', 'a'.repeat(64), a), pb = h.store.receive('u1', 'b'.repeat(64), b);
+ const c = M.jpeg();
+ await assert.rejects(h.store.receive('u1', M.sha(c), M.fakeReq(c)), e => isMediaError(429, 'busy')(e) && e.extra.retryAfter === 5 && e.headers['Retry-After'] === '5');
+ // Another profile is not affected.
+ assert.equal((await h.store.receive('u2', M.sha(c), M.fakeReq(c))).status, 201);
+ a.destroy(new Error('aborted')); b.destroy(new Error('aborted'));
+ await Promise.allSettled([pa, pb]);
+ assert.equal((await h.store.receive('u1', M.sha(c), M.fakeReq(c))).status, 201, 'the slots came back');
+});
+
+test('the disk floor refuses an upload with 507 and holds no reservation afterwards', async t => {
+ const h = setup(t, { minFreeMB: 1e9 });
+ const bytes = M.jpeg();
+ await assert.rejects(h.store.receive('u1', M.sha(bytes), M.fakeReq(bytes)), isMediaError(507, 'storage-full'));
+ assert.deepEqual(h.tmpFiles('u1'), []);
+});
diff --git a/api/test/media-refs.test.js b/api/test/media-refs.test.js
new file mode 100644
index 000000000..22079822c
--- /dev/null
+++ b/api/test/media-refs.test.js
@@ -0,0 +1,31 @@
+/* referencedHashes() decides which stored files the server keeps. The app has its own copy
+ (frontend/src/lib/media-refs.js) deciding which local files the device keeps, and the two must
+ never disagree: a hash the server counts and the device does not is a file one side deletes
+ while the other still shows it. Both are tested against this one fixture. */
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { referencedHashes } from '../media.js';
+
+const FIXTURE = path.join(path.dirname(fileURLToPath(import.meta.url)), 'fixtures', 'media-refs.json');
+const { referencedHashes: cases } = JSON.parse(fs.readFileSync(FIXTURE, 'utf8'));
+
+test('the shared fixture is there and has cases', () => {
+ assert.ok(Array.isArray(cases) && cases.length >= 10);
+});
+
+for (const c of cases) {
+ test(`referencedHashes: ${c.name}`, () => {
+ const got = referencedHashes(c.state);
+ assert.ok(got instanceof Set);
+ assert.deepEqual([...got].sort(), [...c.expect].sort());
+ });
+}
+
+test('never throws, whatever the state is', () => {
+ for (const S of [undefined, 0, '', 'x', [], [1], { customEx: null }, { customEx: [{ media: null }] }, { customEx: [{ media: { poster: 'x' } }] }]) {
+ assert.deepEqual([...referencedHashes(S)], []);
+ }
+});
diff --git a/api/test/media-samples.mjs b/api/test/media-samples.mjs
new file mode 100644
index 000000000..225643dcd
--- /dev/null
+++ b/api/test/media-samples.mjs
@@ -0,0 +1,99 @@
+/* Tiny synthetic media files for the media tests: just enough structure for the magic bytes and
+ the MP4 box walk, built in code so the repo carries no binary fixtures and every test states
+ exactly which bytes it feeds in. Not a *.test.js file, so `npm test` does not run it. */
+import crypto from 'node:crypto';
+import { Readable } from 'node:stream';
+
+export const sha = buf => crypto.createHash('sha256').update(buf).digest('hex');
+const pad = n => crypto.randomBytes(Math.max(0, n));
+
+export const jpeg = (n = 1000) => Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), pad(n - 4)]);
+export const png = (n = 1000) => Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), pad(n - 8)]);
+export const gif = (n = 1000, v = '89a') => Buffer.concat([Buffer.from('GIF' + v, 'latin1'), pad(n - 6)]);
+export function webp(n = 1000) {
+ const b = Buffer.concat([Buffer.from('RIFF', 'latin1'), Buffer.alloc(4), Buffer.from('WEBPVP8 ', 'latin1'), pad(n - 16)]);
+ b.writeUInt32LE(b.length - 8, 4);
+ return b;
+}
+export const svg = () => Buffer.from('<?xml version="1.0"?><svg xmlns="http://www.w3.org/2000/svg" onload="alert(1)"></svg>');
+export const html = () => Buffer.from('<!doctype html><html><script>alert(1)</script></html>');
+export const pdf = () => Buffer.from('%PDF-1.7\n%âãÏÓ\n1 0 obj<<>>endobj\n', 'latin1');
+export const zip = () => Buffer.concat([Buffer.from([0x50, 0x4b, 0x03, 0x04]), pad(100)]);
+
+/* ---- EBML (WebM / Matroska) ---- */
+export function ebml(docType = 'webm', n = 400) {
+ const el = (id, body) => Buffer.concat([Buffer.from(id), Buffer.from([0x80 | body.length]), body]);
+ const kids = Buffer.concat([
+ el([0x42, 0x86], Buffer.from([1])), el([0x42, 0xf7], Buffer.from([1])),
+ el([0x42, 0xf2], Buffer.from([4])), el([0x42, 0xf3], Buffer.from([8])),
+ el([0x42, 0x82], Buffer.from(docType, 'latin1')),
+ el([0x42, 0x87], Buffer.from([4])), el([0x42, 0x85], Buffer.from([2]))
+ ]);
+ const head = Buffer.concat([Buffer.from([0x1a, 0x45, 0xdf, 0xa3, 0x80 | kids.length]), kids]);
+ return Buffer.concat([head, pad(n - head.length)]);
+}
+
+/* ---- ISO-BMFF (MP4 / MOV / HEIC / M4A) ---- */
+export function box(type, ...parts) {
+ const body = Buffer.concat(parts.map(p => (Buffer.isBuffer(p) ? p : Buffer.from(p))));
+ const h = Buffer.alloc(8);
+ h.writeUInt32BE(8 + body.length, 0);
+ h.write(type, 4, 'latin1');
+ return Buffer.concat([h, body]);
+}
+// size = 1 and a 64-bit size after the type, which is how a camera writes an mdat over 4 GB.
+export function box64(type, body) {
+ const h = Buffer.alloc(16);
+ h.writeUInt32BE(1, 0);
+ h.write(type, 4, 'latin1');
+ h.writeBigUInt64BE(BigInt(16 + body.length), 8);
+ return Buffer.concat([h, body]);
+}
+export const ftyp = (brand = 'isom', compat = ['isom', 'mp41']) =>
+ box('ftyp', Buffer.from(brand, 'latin1'), Buffer.alloc(4), Buffer.from(compat.join(''), 'latin1'));
+
+export function mvhd({ version = 0, timescale = 1000, duration = 5000 } = {}) {
+ if (version === 1) {
+ const b = Buffer.alloc(112);
+ b[0] = 1;
+ b.writeUInt32BE(timescale, 20);
+ b.writeBigUInt64BE(BigInt(duration), 24);
+ return box('mvhd', b);
+ }
+ const b = Buffer.alloc(100);
+ b.writeUInt32BE(timescale, 12);
+ b.writeUInt32BE(duration >>> 0, 16);
+ return box('mvhd', b);
+}
+export function mehd(duration) {
+ const b = Buffer.alloc(8);
+ b.writeUInt32BE(duration, 4);
+ return box('mehd', b);
+}
+
+/**
+ * A structurally valid MP4/MOV: ftyp, then moov (mvhd + an empty trak) and mdat in either order.
+ * `seconds` is written as mvhd's duration at `timescale`; `moovPad` inflates moov with a `free`
+ * box, `mdatBytes` sizes the media data.
+ */
+export function mp4({ brand = 'isom', seconds = 5, timescale = 1000, version = 0, moovAtEnd = false, mdatBytes = 2000, moovPad = 0, extraMoov = [], bigMdat = false } = {}) {
+ const moov = box('moov', mvhd({ version, timescale, duration: Math.round(seconds * timescale) }),
+ box('trak', box('tkhd', Buffer.alloc(84))), ...extraMoov, ...(moovPad ? [box('free', Buffer.alloc(moovPad))] : []));
+ const mdat = bigMdat ? box64('mdat', pad(mdatBytes)) : box('mdat', pad(mdatBytes));
+ return Buffer.concat(moovAtEnd ? [ftyp(brand), mdat, moov] : [ftyp(brand), moov, mdat]);
+}
+
+/* ---- a request stand-in for receive() ---- */
+/**
+ * What receive() reads off a request: `headers` and a byte stream. `bytes` is sent in `chunk`
+ * pieces; `declare` is the Content-Type, `length` the Content-Length (true = the real length,
+ * a number = that claim, null = none, as with chunked encoding).
+ */
+export function fakeReq(bytes, { declare = 'image/jpeg', length = true, chunk = 64 * 1024 } = {}) {
+ const parts = [];
+ for (let i = 0; i < bytes.length; i += chunk) parts.push(bytes.subarray(i, i + chunk));
+ const req = Readable.from(parts, { objectMode: false });
+ req.headers = { 'content-type': declare };
+ if (length !== null) req.headers['content-length'] = String(length === true ? bytes.length : length);
+ return req;
+}
diff --git a/api/test/media-sniff.test.js b/api/test/media-sniff.test.js
new file mode 100644
index 000000000..15f12e71c
--- /dev/null
+++ b/api/test/media-sniff.test.js
@@ -0,0 +1,112 @@
+/* What the server stores is decided by the first bytes of a file and nothing else: not its name,
+ not the Content-Type the client declared. Seven types pass; SVG and HTML — the two that would
+ run in the app's origin if a browser were ever talked into rendering them — and everything
+ else do not. mp4Info is the one place the server looks deeper, to read a video's length off
+ its headers without loading it. */
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { sniffMedia, mp4Info, MEDIA_TYPES } from '../media.js';
+import * as M from './media-samples.mjs';
+
+const head = b => b.subarray(0, 64);
+const kindOf = b => sniffMedia(head(b));
+
+test('each of the seven stored types is recognised from its magic bytes', () => {
+ const cases = [
+ [M.jpeg(), 'image/jpeg', 'image', 'still', 'jpg'],
+ [M.png(), 'image/png', 'image', 'still', 'png'],
+ [M.webp(), 'image/webp', 'image', 'still', 'webp'],
+ [M.gif(), 'image/gif', 'gif', 'still', 'gif'],
+ [M.gif(1000, '87a'), 'image/gif', 'gif', 'still', 'gif'],
+ [M.mp4(), 'video/mp4', 'video', 'video', 'mp4'],
+ [M.mp4({ brand: 'qt ' }), 'video/quicktime', 'video', 'video', 'mov'],
+ [M.ebml('webm'), 'video/webm', 'video', 'video', 'webm']
+ ];
+ for (const [bytes, mime, kind, category, ext] of cases) {
+ assert.deepEqual(kindOf(bytes), { mime, kind, category, ext }, mime);
+ }
+ assert.equal(Object.keys(MEDIA_TYPES).length, 7);
+});
+
+test('every MP4 brand on the list is a video, and a brand nobody vouched for is not', () => {
+ for (const brand of ['isom', 'iso2', 'iso3', 'iso4', 'iso5', 'iso6', 'mp41', 'mp42', 'avc1', 'M4V ', 'M4VP', 'dash', 'mmp4', 'MSNV', '3gp4', '3gp5', '3gp6', '3g2a']) {
+ assert.equal(kindOf(M.mp4({ brand }))?.mime, 'video/mp4', brand);
+ }
+ for (const brand of ['abcd', 'iso9', 'mp71', 'crx ']) assert.equal(kindOf(M.mp4({ brand })), null, brand);
+});
+
+test('HEIC and AVIF stills are refused — the app re-encodes them, it never uploads them', () => {
+ for (const brand of ['heic', 'heix', 'heim', 'heis', 'hevc', 'mif1', 'msf1', 'avif', 'avis']) {
+ assert.equal(kindOf(M.mp4({ brand })), null, brand);
+ }
+});
+
+test('SVG, HTML, PDF, ZIP, M4A audio and Matroska are refused', () => {
+ assert.equal(kindOf(M.svg()), null, 'svg');
+ assert.equal(kindOf(M.html()), null, 'html');
+ assert.equal(kindOf(M.pdf()), null, 'pdf');
+ assert.equal(kindOf(M.zip()), null, 'zip');
+ assert.equal(kindOf(M.mp4({ brand: 'M4A ' })), null, 'm4a');
+ assert.equal(kindOf(M.ebml('matroska')), null, 'matroska');
+ assert.equal(kindOf(Buffer.from('BM' + 'x'.repeat(60))), null, 'bmp');
+ assert.equal(kindOf(Buffer.from([0x49, 0x49, 0x2a, 0x00, 1, 2, 3, 4])), null, 'tiff');
+ assert.equal(kindOf(Buffer.from([0, 0, 1, 0, 1, 0])), null, 'ico');
+ assert.equal(kindOf(Buffer.alloc(0)), null, 'empty');
+ assert.equal(kindOf(Buffer.from([0xff, 0xd8])), null, 'truncated jpeg');
+});
+
+/* ---------------- mp4Info ---------------- */
+
+const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'media-sniff-'));
+test.after(() => fs.rmSync(dir, { recursive: true, force: true }));
+let n = 0;
+function info(bytes) {
+ const f = path.join(dir, `f${n++}.mp4`);
+ fs.writeFileSync(f, bytes);
+ const fd = fs.openSync(f, 'r');
+ try { return mp4Info(fd, bytes.length); } finally { fs.closeSync(fd); }
+}
+
+test('mp4Info reads the length from mvhd, version 0 and version 1', () => {
+ assert.deepEqual(info(M.mp4({ seconds: 12.5 })), { brand: 'isom', durationSec: 12.5 });
+ assert.deepEqual(info(M.mp4({ version: 1, seconds: 42, timescale: 90000 })), { brand: 'isom', durationSec: 42 });
+ assert.deepEqual(info(M.mp4({ brand: 'qt ', seconds: 61, timescale: 600 })), { brand: 'qt ', durationSec: 61 });
+});
+
+test('mp4Info finds a moov that sits after the media data, even behind a 64-bit mdat', () => {
+ assert.equal(info(M.mp4({ moovAtEnd: true, seconds: 30, mdatBytes: 200000 })).durationSec, 30);
+ assert.equal(info(M.mp4({ moovAtEnd: true, seconds: 7, bigMdat: true })).durationSec, 7);
+});
+
+test('a fragmented file with no length in mvhd is read from mehd, and one with neither is unknown', () => {
+ const frag = M.mp4({ seconds: 0, extraMoov: [M.box('mvex', M.mehd(95000))] });
+ assert.equal(info(frag).durationSec, 95);
+ assert.equal(info(M.mp4({ seconds: 0 })).durationSec, null);
+});
+
+test('mp4Info refuses what is not a well-formed file', () => {
+ const noMoov = Buffer.concat([M.ftyp(), M.box('mdat', Buffer.alloc(100))]);
+ assert.equal(info(noMoov), null, 'no moov');
+ const noMvhd = Buffer.concat([M.ftyp(), M.box('moov', M.box('trak', Buffer.alloc(20)))]);
+ assert.equal(info(noMvhd), null, 'no mvhd');
+ assert.equal(info(M.mp4({ timescale: 0 })), null, 'zero timescale');
+ assert.equal(info(Buffer.concat([M.box('free', Buffer.alloc(8)), M.mp4()])), null, 'ftyp not first');
+ // A moov over 8 MB: no real clip has one, and walking it is what an attacker would want.
+ assert.equal(info(M.mp4({ moovPad: 8 * 1024 * 1024 + 10 })), null, 'moov > 8 MB');
+ // A box that claims more bytes than its parent has.
+ const lying = M.mp4();
+ lying.writeUInt32BE(lying.length * 2, M.ftyp().length);
+ assert.equal(info(lying), null, 'box overruns the file');
+ // A size smaller than its own header.
+ const tiny = M.mp4();
+ tiny.writeUInt32BE(4, M.ftyp().length);
+ assert.equal(info(tiny), null, 'box smaller than its header');
+});
+
+test('mp4Info gives up after 10k boxes instead of walking forever', () => {
+ const many = Buffer.concat([M.ftyp(), ...Array.from({ length: 10050 }, () => M.box('free')), M.box('moov', M.mvhd())]);
+ assert.equal(info(many), null);
+});