Open-source gym & body-weight tracker, now in the App Store (iPhone + Apple Watch) and as an Android APK. Plan routines, log workouts, see which muscles are trained, import from Strong/Hevy/FitNotes. Self-host it or use it on the phone alone.
JavaScript apple-watchbodyweightdockerfitnessfitness-trackergymhealthiosmcpnodejspasskeysprogressive-web-applicationspwareactself-hostedself-hostingvitewebauthnweightliftingworkout-tracker
API: a store for the photos and videos of custom exercises — per profile, named by their sha256, streamed to disk, and never deleted for anything missing
Duarte Santos424845df2ec3
diff --git a/api/media.js b/api/media.jsnew file mode 100644index 000000000..489227fbb--- /dev/null+++ b/api/media.js@@ -0,0 +1,724 @@+/* Photos, GIFs and videos of a person's own exercises: the server half.+ *+ * A custom exercise in the state carries a MediaRef (sha256, kind, mime, size, dimensions, a+ * poster ref) and never the bytes. The bytes live here, per profile and named by their hash:+ *+ * DATA_DIR/uploads/<uid>/<sha256>.<ext> 0700 directories, 0600 files+ * DATA_DIR/uploads/<uid>/.gc.json { hash: unreferencedSinceMs }+ * DATA_DIR/uploads/<uid>/.tmp/<random> uploads in flight; never outlive their request+ *+ * What this module promises, and why each promise is shaped the way it is:+ *+ * - It never decodes media. The only bytes it interprets are the first 64 (magic numbers) and,+ * for MP4/MOV, the box headers down to mvhd. No image library means no decompression bomb,+ * no codec parser and no dependency; the metadata scrub and re-encode happen on the device+ * that picked the file.+ * - It never buffers an upload. The production API runs with a 128 MB memory limit and a video+ * may be 40 MB, so bytes go from the socket through a hash to a temp file and nowhere else.+ * - It never deletes because something is missing. A state file that cannot be read, a user+ * that is not in db.json (a db.json that failed to parse boots with no users at all) and an+ * unreadable .gc.json all mean "keep everything". Blobs only go when the profile's own+ * readable state has not referenced them for the whole grace period, or when the profile+ * itself is deleted. Devices that still hold a blob the server dropped upload it again.+ * - The directory is the access check. Nothing here takes a uid from a request; every lookup+ * is in the caller's own folder, so there is no way to learn whether someone else has a file.+ */+import crypto from 'node:crypto';+import fs from 'node:fs';+import path from 'node:path';++export const HASH_RE = /^[0-9a-f]{64}$/;+const MB = 1024 * 1024; // every *_MB setting means MiB; the client uses the same unit+const HOUR = 3600000;+const DAY = 24 * HOUR;+const HEAD_BYTES = 64; // enough for every magic number below, the WebM DocType included+const MOOV_MAX = 8 * MB; // a real moov for a 60 s clip is well under 1 MB+const MAX_BOXES = 10000;+export const MAX_INFLIGHT = 2; // concurrent uploads per profile++/* The seven types that are ever stored. `category` is what the client may not lie about: a+ JPEG declared as a GIF is still a still image and is stored as the JPEG it is, but a video+ declared as an image would slip past the image cap, so that mismatch is refused. */+export const MEDIA_TYPES = {+ 'image/jpeg': { kind: 'image', category: 'still', ext: 'jpg' },+ 'image/png': { kind: 'image', category: 'still', ext: 'png' },+ 'image/webp': { kind: 'image', category: 'still', ext: 'webp' },+ 'image/gif': { kind: 'gif', category: 'still', ext: 'gif' },+ 'video/mp4': { kind: 'video', category: 'video', ext: 'mp4' },+ 'video/quicktime': { kind: 'video', category: 'video', ext: 'mov' },+ 'video/webm': { kind: 'video', category: 'video', ext: 'webm' }+};+const EXT_MIME = Object.fromEntries(Object.entries(MEDIA_TYPES).map(([mime, t]) => [t.ext, mime]));+const FILE_RE = /^([0-9a-f]{64})\.(jpg|png|webp|gif|mp4|mov|webm)$/;++/* ---------------------------------------------------------------- limits */++const OFF = /^(0|false|no|off)$/i;+/** The instance's media settings, read from the environment (docs/SELF_HOSTING.md). Every MB+ * value is MiB and may be fractional. A value that does not parse falls back to its default+ * rather than to 0: a typo must not turn the quota off or refuse every upload. */+export function mediaLimits(env = process.env) {+ const num = (key, def, min) => {+ const raw = env[key];+ if (raw == null || String(raw).trim() === '') return def;+ const n = Number(raw);+ return Number.isFinite(n) && n >= min ? n : def;+ };+ return {+ enabled: !OFF.test(String(env.MEDIA_UPLOADS || '').trim()),+ quotaMB: num('MEDIA_QUOTA_MB', 200, 0), // 0 = no cap+ imageMB: num('MEDIA_IMAGE_MAX_MB', 2, 0.001),+ gifMB: num('MEDIA_GIF_MAX_MB', 8, 0.001),+ videoMB: num('MEDIA_VIDEO_MAX_MB', 40, 0.001),+ videoSec: num('MEDIA_VIDEO_MAX_SEC', 60, 1),+ gcGraceDays: num('MEDIA_GC_GRACE_DAYS', 14, 0),+ uploadsPerHour: Math.floor(num('MEDIA_UPLOADS_PER_HOUR', 600, 1)),+ minFreeMB: num('MEDIA_MIN_FREE_MB', 512, 0) // 0 = no floor+ };+}+/** The block GET /api/config hands out. Public: the caps are not a secret, and the editor needs+ * them before the first upload to refuse a file the server would refuse anyway. */+export const mediaConfig = l => ({ imageMB: l.imageMB, gifMB: l.gifMB, videoMB: l.videoMB, videoSec: l.videoSec, quotaMB: l.quotaMB });++/* ---------------------------------------------------------------- errors */++const MESSAGES = {+ 'bad-request': 'bad request',+ 'media-type': 'that file type is not accepted',+ 'media-too-large': 'that file is too large',+ 'media-quota': 'your space for photos and videos is full',+ 'media-invalid': 'that video could not be read',+ 'media-too-long': 'that video is too long',+ 'media-missing': 'no such file',+ 'hash-mismatch': 'the file does not match its name',+ 'storage-full': 'the server is running out of disk space',+ busy: 'too many uploads at once — try again in a moment',+ locked: 'too many uploads — try again later',+ timeout: 'the upload stalled'+};+/** A refusal the client caused or can act on. The server's catch-all answers it as+ * `{error, code, ...extra}` with its status and headers, and does not log it: a refused file+ * is not a server error, and a stack trace per refusal would bury the real ones. */+export class MediaError extends Error {+ constructor(status, code, extra = {}, headers = undefined) {+ super(MESSAGES[code] || code);+ this.status = status;+ this.code = code;+ this.extra = extra;+ this.headers = headers;+ }+}++/* ---------------------------------------------------------------- sniffing */++const MP4_BRANDS = new Set([+ 'isom', 'iso2', 'iso3', 'iso4', 'iso5', 'iso6', 'mp41', 'mp42', 'avc1', 'M4V ', 'M4VP',+ 'dash', 'mmp4', 'MSNV', '3gp4', '3gp5', '3gp6', '3g2a'+]);++// An EBML variable-length integer at b[i]. IDs keep their length marker, sizes drop it.+function vint(b, i, keepMarker) {+ if (i >= b.length || b[i] === 0) return null;+ const first = b[i];+ let len = 1;+ while (!(first & (0x80 >> (len - 1)))) len++;+ if (i + len > b.length) return null;+ let v = keepMarker ? first : first & (0xff >> len);+ for (let k = 1; k < len; k++) v = v * 256 + b[i + k];+ return { value: v, len };+}+// The DocType of an EBML header ('webm', 'matroska'), read off the header's children.+function ebmlDocType(b) {+ const hs = vint(b, 4, false);+ if (!hs) return null;+ let i = 4 + hs.len;+ const end = Math.min(b.length, i + hs.value);+ while (i < end) {+ const id = vint(b, i, true);+ const sz = id && vint(b, i + id.len, false);+ if (!sz) return null;+ const body = i + id.len + sz.len;+ if (id.value === 0x4282) {+ if (body + sz.value > b.length) return null;+ return Buffer.from(b.subarray(body, body + sz.value)).toString('latin1').replace(/\0+$/, '');+ }+ i = body + sz.value;+ }+ return null;+}++/**+ * What a file is, from its first bytes alone — never from its name or a declared type.+ * Returns { mime, kind, category, ext } or null. The client's sniffKind() follows the same rules+ * and additionally reads HEIC/AVIF, which it only ever re-encodes; the server refuses them.+ */+export function sniffMedia(head) {+ const b = Buffer.isBuffer(head) ? head : Buffer.from(head || []);+ const as = mime => ({ mime, ...MEDIA_TYPES[mime] });+ const at = (off, ...bytes) => bytes.every((x, i) => b[off + i] === x);+ const ascii = (off, len) => b.length >= off + len ? b.toString('latin1', off, off + len) : '';+ if (at(0, 0xff, 0xd8, 0xff)) return as('image/jpeg');+ if (at(0, 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a)) return as('image/png');+ if (ascii(0, 6) === 'GIF87a' || ascii(0, 6) === 'GIF89a') return as('image/gif');+ if (ascii(0, 4) === 'RIFF' && ascii(8, 4) === 'WEBP') return as('image/webp');+ if (ascii(4, 4) === 'ftyp') {+ const brand = ascii(8, 4);+ if (brand === 'qt ') return as('video/quicktime');+ if (MP4_BRANDS.has(brand)) return as('video/mp4');+ return null; // HEIC/AVIF stills, M4A audio and every brand nobody vouched for+ }+ if (at(0, 0x1a, 0x45, 0xdf, 0xa3)) return ebmlDocType(b) === 'webm' ? as('video/webm') : null;+ return null;+}++/* ---------------------------------------------------------------- MP4 / MOV */++function readAt(fd, pos, len) {+ const buf = Buffer.alloc(len);+ return fs.readSync(fd, buf, 0, len, pos) === len ? buf : null;+}+// The box whose header starts at `pos`, which must end by `end`. size 1 = a 64-bit size follows,+// size 0 = the box runs to `end`. Anything that would reach outside its parent is a broken file.+function boxAt(fd, pos, end) {+ if (end - pos < 8) return null;+ const h = readAt(fd, pos, Math.min(16, end - pos));+ if (!h) return null;+ let size = h.readUInt32BE(0);+ let headerLen = 8;+ if (size === 1) {+ if (h.length < 16) return null;+ size = u64(h, 8);+ if (size === Number.MAX_SAFE_INTEGER) return null;+ headerLen = 16;+ } else if (size === 0) size = end - pos;+ if (size < headerLen || pos + size > end) return null;+ return { type: h.toString('latin1', 4, 8), body: pos + headerLen, bodyLen: size - headerLen, end: pos + size };+}+// A 64-bit field as a Number, clamped: nothing in a real file comes near 2^53, and all ones (the+// "unknown" marker) has to stay recognisable after the conversion.+const u64 = (buf, off) => { const v = buf.readBigUInt64BE(off); return v > BigInt(Number.MAX_SAFE_INTEGER) ? Number.MAX_SAFE_INTEGER : Number(v); };+// A duration field that says "unknown": zero (fragmented files) or all ones.+const unknownDur = (d, version) => d === 0 || d === (version === 1 ? Number.MAX_SAFE_INTEGER : 0xffffffff);++/**+ * The brand and length of an MP4/MOV, read from box headers with positioned reads — the moov+ * may sit at the very end (a file written without "fast start"), and the file is never loaded.+ * Returns { brand, durationSec } — durationSec null when the file does not say — or null for+ * anything that is not a well-formed file: no ftyp first, no moov, no mvhd, a zero timescale, a+ * box that overruns its parent, more than 10k boxes, or a moov over 8 MB.+ */+export function mp4Info(fd, size) {+ try {+ let pos = 0, count = 0, brand = null, moov = null;+ while (pos < size) {+ if (++count > MAX_BOXES) return null;+ const b = boxAt(fd, pos, size);+ if (!b) return null;+ if (count === 1) {+ if (b.type !== 'ftyp' || b.bodyLen < 4) return null;+ brand = readAt(fd, b.body, 4).toString('latin1');+ }+ if (b.type === 'moov') { moov = b; break; }+ pos = b.end;+ }+ if (!moov || moov.bodyLen + 8 > MOOV_MAX) return null;+ let timescale = 0, dur = null, fragDur = null, sawMvhd = false;+ pos = moov.body;+ while (pos < moov.end) {+ if (++count > MAX_BOXES) return null;+ const b = boxAt(fd, pos, moov.end);+ if (!b) return null;+ if (b.type === 'mvhd' && !sawMvhd) {+ sawMvhd = true;+ const v = readAt(fd, b.body, Math.min(b.bodyLen, 32));+ if (!v || v.length < 20) return null;+ if (v[0] === 1) {+ if (v.length < 32) return null;+ timescale = v.readUInt32BE(20);+ dur = u64(v, 24);+ if (unknownDur(dur, 1)) dur = null;+ } else {+ timescale = v.readUInt32BE(12);+ dur = v.readUInt32BE(16);+ if (unknownDur(dur, 0)) dur = null;+ }+ } else if (b.type === 'mvex') {+ // A fragmented file (Safari's MediaRecorder, some editors) leaves mvhd at 0 and states+ // the whole length here, in mehd, in the same timescale.+ let p = b.body;+ while (p < b.end) {+ if (++count > MAX_BOXES) return null;+ const c = boxAt(fd, p, b.end);+ if (!c) return null;+ if (c.type === 'mehd') {+ const v = readAt(fd, c.body, Math.min(c.bodyLen, 12));+ if (v && v.length >= 8) {+ const wide = v[0] === 1 && v.length >= 12;+ const d = wide ? u64(v, 4) : v.readUInt32BE(4);+ if (!unknownDur(d, wide ? 1 : 0)) fragDur = d;+ }+ }+ p = c.end;+ }+ }+ pos = b.end;+ }+ if (!sawMvhd || !timescale) return null;+ const ticks = dur ?? fragDur;+ return { brand, durationSec: ticks == null ? null : Math.round((ticks / timescale) * 1000) / 1000 };+ } catch {+ return null;+ }+}++/* ---------------------------------------------------------------- references */++/**+ * Every blob a state refers to: customEx[].media.hash and customEx[].media.poster.hash.+ * Deliberately loose — each field counts on its own as long as it is a well-formed hash, even+ * when the rest of the ref would not pass the client's normalizeMediaRef. This set decides what+ * is KEPT, so erring towards more is the safe direction. frontend/src/lib/media-refs.js has to+ * agree with it; api/test/fixtures/media-refs.json pins both to the same answers.+ */+export function referencedHashes(state) {+ const out = new Set();+ const list = state && typeof state === 'object' && Array.isArray(state.customEx) ? state.customEx : [];+ const add = r => { if (r && typeof r === 'object' && typeof r.hash === 'string' && HASH_RE.test(r.hash)) out.add(r.hash); };+ for (const c of list) {+ const m = c && typeof c === 'object' ? c.media : null;+ if (!m || typeof m !== 'object') continue;+ add(m);+ add(m.poster);+ }+ return out;+}++/* ---------------------------------------------------------------- the store */++const round1 = n => Math.round(n * 10) / 10;+const contentType = req => String(req.headers?.['content-type'] || '').split(';')[0].trim().toLowerCase();+function contentLength(req) {+ const raw = req.headers?.['content-length'];+ if (raw == null || raw === '') return null;+ const n = Number(raw);+ return Number.isSafeInteger(n) && n >= 0 ? n : null;+}+// Reads and discards what is left of a refused upload, so the answer reaches the client instead+// of a connection reset (closing a socket with unread bytes makes the kernel send RST, and a+// client that sees it before parsing the answer reports a dropped connection — see readBody in+// server.js). A client that keeps sending past `limit` is not a mistaken one and is cut off.+function drain(req, limit) {+ if (req.readableEnded || req.destroyed) return;+ let seen = 0;+ req.on('data', d => { seen += d.length; if (seen > limit) req.destroy(); });+ req.on('error', () => {});+ req.resume();+}++/**+ * Streams `req` into a new file at `file`, hashing and counting on the way. Resolves with+ * { size, sha, head } once every byte is on disk and the file is closed. Rejects with a+ * MediaError past `max` bytes (413) or after `idleMs` without a byte (408), with the socket's+ * own error (marked clientGone) when the client hangs up, and with the disk's error otherwise.+ * On every rejection the write stream has closed before the promise settles, so the caller can+ * unlink the file without racing a late open().+ */+function streamToFile(req, file, { max, maxMB, idleMs }) {+ return new Promise((resolve, reject) => {+ const hash = crypto.createHash('sha256');+ const head = [];+ let headLen = 0, size = 0, ended = false, settled = false, timer = null;+ const out = fs.createWriteStream(file, { flags: 'wx', mode: 0o600, highWaterMark: 256 * 1024 });+ const arm = () => {+ clearTimeout(timer);+ timer = setTimeout(() => settle(new MediaError(408, 'timeout', {}, { Connection: 'close' })), idleMs);+ };+ const detach = () => {+ clearTimeout(timer);+ req.off('data', onData); req.off('end', onEnd); req.off('error', onReqError); req.off('close', onClose);+ out.off('drain', onDrain);+ };+ function settle(err, val) {+ if (settled) return;+ settled = true;+ detach();+ if (!err) return resolve(val);+ if (err.code === 'ENOSPC') err = new MediaError(507, 'storage-full');+ if (out.closed) return reject(err);+ out.once('close', () => reject(err));+ out.destroy();+ }+ function onData(chunk) {+ arm();+ size += chunk.length;+ if (size > max) return settle(new MediaError(413, 'media-too-large', { maxMB }));+ hash.update(chunk);+ if (headLen < HEAD_BYTES) {+ // Copied, so the head does not keep a whole 64 KB socket chunk alive.+ const part = Buffer.from(chunk.subarray(0, HEAD_BYTES - headLen));+ head.push(part);+ headLen += part.length;+ }+ if (!out.write(chunk)) req.pause();+ }+ function onDrain() { if (!settled && !ended) { arm(); req.resume(); } }+ function onEnd() {+ ended = true;+ clearTimeout(timer);+ out.end();+ out.once('close', () => {+ if (settled) return;+ settled = true;+ detach();+ resolve({ size, sha: hash.digest('hex'), head: Buffer.concat(head) });+ });+ }+ function onReqError(e) { settle(Object.assign(e || new Error('aborted'), { clientGone: true })); }+ // 'close' before 'end' is a client that went away without an error event (an aborted+ // request on some Node versions); after 'end' it is the ordinary end of the request.+ function onClose() { if (!ended) settle(Object.assign(new Error('client went away mid-upload'), { clientGone: true })); }+ out.on('error', e => settle(e));+ out.on('drain', onDrain);+ req.on('data', onData);+ req.on('end', onEnd);+ req.on('error', onReqError);+ req.on('close', onClose);+ arm();+ req.resume();+ });+}++/**+ * The per-instance media store. `dir` is DATA_DIR/uploads. `readState(uid)` returns the+ * profile's stored state, or null when there is none or it does not parse — the store sweeps+ * nobody it gets null for. `now` is injectable so the tests can walk the clock.+ */+export function createMediaStore({ dir, limits, now = Date.now, readState = () => null, idleMs = 60000, log = console } = {}) {+ const L = { ...mediaLimits({}), ...(limits || {}) };+ const quotaBytes = L.quotaMB > 0 ? Math.round(L.quotaMB * MB) : 0;+ const capMB = kind => (kind === 'video' ? L.videoMB : kind === 'gif' ? L.gifMB : L.imageMB);+ const capOf = kind => Math.round(capMB(kind) * MB);+ const maxCap = Math.max(capOf('image'), capOf('gif'), capOf('video'));+ const users = new Map(); // safe uid -> { dir, bytes, count, reserved, inflight, hashes }+ const orphansLogged = new Set();++ // The same sanitising as server.js stateFile(). An id that sanitises to nothing would name+ // uploads/ itself, and removeUser() would then delete everybody's files.+ function safe(uid) {+ const s = String(uid ?? '').replace(/[^a-zA-Z0-9_-]/g, '');+ if (!s) throw new Error('media: a profile id is required');+ return s;+ }+ function ensureDir(p) {+ fs.mkdirSync(p, { recursive: true, mode: 0o700 });+ // mkdir's mode is filtered by the umask and does nothing for a directory that already+ // existed; the Coach runtime's uid must not be able to list anyone's files.+ try { fs.chmodSync(p, 0o700); } catch { /* a host filesystem that refuses chmod */ }+ }++ // Built once per profile from one readdir, then kept current by every write and delete, so+ // a state push never lists a directory.+ function entry(uid) {+ const id = safe(uid);+ let e = users.get(id);+ if (e) return e;+ e = { id, dir: path.join(dir, id), bytes: 0, count: 0, reserved: 0, inflight: 0, hashes: new Map() };+ let names = [];+ try { names = fs.readdirSync(e.dir, { withFileTypes: true }); } catch { /* nothing uploaded yet */ }+ for (const d of names) {+ const m = FILE_RE.exec(d.name);+ if (!m || !d.isFile() || e.hashes.has(m[1])) continue;+ try {+ const size = fs.statSync(path.join(e.dir, d.name)).size;+ e.hashes.set(m[1], { ext: m[2], size });+ e.bytes += size;+ e.count++;+ } catch { /* went away between readdir and stat */ }+ }+ users.set(id, e);+ return e;+ }+ function forget(e, hash) {+ const f = e.hashes.get(hash);+ if (!f) return;+ e.hashes.delete(hash);+ e.bytes -= f.size;+ e.count--;+ }+ const usageOf = e => ({ bytes: e.bytes, count: e.count, quotaBytes });++ /* .gc.json: when each stored hash was last seen leaving the state. Read fresh every time (it+ is a few hundred bytes), so what is on disk is the only truth. Unreadable = no marks: the+ grace starts over, which can only ever keep a file longer. */+ const gcFile = e => path.join(e.dir, '.gc.json');+ function readMarks(e) {+ try {+ const raw = JSON.parse(fs.readFileSync(gcFile(e), 'utf8'));+ if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return {};+ const out = {};+ for (const [h, t] of Object.entries(raw)) if (HASH_RE.test(h) && Number.isFinite(t)) out[h] = t;+ return out;+ } catch { return {}; }+ }+ function writeMarks(e, marks) {+ const f = gcFile(e), tmp = f + '.tmp';+ try {+ fs.writeFileSync(tmp, JSON.stringify(marks), { mode: 0o600 });+ fs.renameSync(tmp, f);+ } catch (err) {+ try { fs.unlinkSync(tmp); } catch { /* never written */ }+ throw err;+ }+ }+ // Brings the marks in line with `refs`: a referenced hash loses its mark, an unreferenced one+ // without a mark gets `t`, marks for files that are gone are dropped. Returns whether anything+ // changed.+ function reconcile(e, refs, marks, t) {+ let changed = false;+ for (const h of e.hashes.keys()) {+ if (refs.has(h)) { if (h in marks) { delete marks[h]; changed = true; } }+ else if (!(h in marks)) { marks[h] = t; changed = true; }+ }+ for (const h of Object.keys(marks)) if (!e.hashes.has(h)) { delete marks[h]; changed = true; }+ return changed;+ }+ function stateOf(uid) {+ let S = null;+ try { S = readState(uid); } catch { S = null; }+ return S && typeof S === 'object' && !Array.isArray(S) ? S : null;+ }++ function noteState(uid, state) {+ const id = safe(uid);+ // Nothing to do for a profile that never uploaded — and that is the common case, so it+ // costs one stat per profile per process, not a readdir per push.+ if (!users.has(id) && !fs.existsSync(path.join(dir, id))) return false;+ const e = entry(uid);+ if (!e.hashes.size) return false;+ const marks = readMarks(e);+ const changed = reconcile(e, referencedHashes(state), marks, now());+ if (changed) writeMarks(e, marks);+ return changed;+ }++ function sweep(uid, { graceMs = L.gcGraceDays * DAY } = {}) {+ const e = entry(uid);+ const res = { removed: 0, freedBytes: 0, skipped: false };+ if (!e.hashes.size) return res;+ const S = stateOf(uid);+ if (!S) { res.skipped = true; return res; } // never infer anything from a missing state+ const refs = referencedHashes(S);+ const marks = readMarks(e);+ const t = now();+ let changed = reconcile(e, refs, marks, t);+ for (const [h, f] of [...e.hashes]) {+ if (refs.has(h)) continue;+ if (graceMs > 0 && !(t - marks[h] >= graceMs)) continue;+ try { fs.unlinkSync(path.join(e.dir, `${h}.${f.ext}`)); }+ catch (err) { if (err.code !== 'ENOENT') { log.error('media: could not delete', e.id, h, err.message); continue; } }+ forget(e, h);+ delete marks[h];+ changed = true;+ res.removed++;+ res.freedBytes += f.size;+ }+ if (changed) {+ try { writeMarks(e, marks); } catch (err) { log.error('media: could not write .gc.json for', e.id, err.message); }+ }+ return res;+ }++ // Temp files of uploads that died with the process, or that a crash between write and rename+ // left behind. `olderThan` 0 = all of them (boot, when nothing can be in flight).+ function cleanTmpOf(userDir, olderThan) {+ const t = now();+ const tmpDir = path.join(userDir, '.tmp');+ let names = [];+ try { names = fs.readdirSync(tmpDir); } catch { /* none */ }+ let n = 0;+ for (const name of names) {+ const p = path.join(tmpDir, name);+ try {+ if (olderThan > 0 && t - fs.statSync(p).mtimeMs < olderThan) continue;+ fs.rmSync(p, { recursive: true, force: true });+ n++;+ } catch { /* gone already */ }+ }+ try {+ const g = path.join(userDir, '.gc.json.tmp');+ if (olderThan === 0 || t - fs.statSync(g).mtimeMs >= olderThan) fs.unlinkSync(g);+ } catch { /* none */ }+ return n;+ }+ const userDirs = () => {+ try { return fs.readdirSync(dir, { withFileTypes: true }).filter(d => d.isDirectory() && !d.name.startsWith('.')).map(d => d.name); }+ catch { return []; }+ };++ function markIfUnreferenced(uid, e, hash) {+ const S = stateOf(uid);+ if (S && referencedHashes(S).has(hash)) return;+ const marks = readMarks(e);+ marks[hash] = now();+ writeMarks(e, marks);+ }+ // A device just vouched for a file the server already has. If it was on its way out, the+ // grace starts over: the state push that references it is usually a second behind.+ function touch(e, hash) {+ try {+ const marks = readMarks(e);+ if (!(hash in marks)) return;+ marks[hash] = now();+ writeMarks(e, marks);+ } catch (err) { log.error('media: could not write .gc.json for', e.id, err.message); }+ }+ function freeBytes(p) {+ try { const s = fs.statfsSync(p); return Number(s.bavail) * Number(s.bsize); }+ catch { return null; } // a filesystem that cannot say: no floor rather than no uploads+ }+ const existed = (hash, f) => ({ status: 200, body: { ok: true, hash, mime: EXT_MIME[f.ext], size: f.size, existed: true } });++ async function receive(uid, hash, req) {+ if (!HASH_RE.test(String(hash))) { drain(req, maxCap); throw new MediaError(400, 'bad-request'); }+ const e = entry(uid);+ if (e.inflight >= MAX_INFLIGHT) {+ drain(req, 2 * maxCap);+ throw new MediaError(429, 'busy', { retryAfter: 5 }, { 'Retry-After': '5' });+ }+ e.inflight++;+ let reserved = 0, tmp = null, cap = maxCap;+ try {+ const declared = MEDIA_TYPES[contentType(req)];+ if (!declared) throw new MediaError(415, 'media-type');+ cap = capOf(declared.kind);+ const len = contentLength(req);+ if (len != null && len > cap) throw new MediaError(413, 'media-too-large', { maxMB: capMB(declared.kind) });+ const have = e.hashes.get(hash);+ if (have) { drain(req, 2 * cap); touch(e, hash); return existed(hash, have); }++ // Reserved before a byte is read, so two uploads at once cannot both fit into the space+ // that is only left for one.+ const need = len ?? cap;+ if (quotaBytes && e.bytes + e.reserved + need > quotaBytes) {+ // Space the person already let go of may still be sitting out its grace period; under+ // pressure it gets an hour instead of two weeks.+ try { sweep(uid, { graceMs: HOUR }); } catch (err) { log.error('media: quota sweep failed for', e.id, err.message); }+ if (e.bytes + e.reserved + need > quotaBytes) {+ throw new MediaError(413, 'media-quota', { usedMB: round1(e.bytes / MB), quotaMB: L.quotaMB });+ }+ }+ e.reserved += need;+ reserved = need;++ ensureDir(dir);+ ensureDir(e.dir);+ ensureDir(path.join(e.dir, '.tmp'));+ if (L.minFreeMB > 0) {+ const free = freeBytes(e.dir);+ if (free != null && free - need < L.minFreeMB * MB) throw new MediaError(507, 'storage-full');+ }++ tmp = path.join(e.dir, '.tmp', crypto.randomBytes(12).toString('hex'));+ const got = await streamToFile(req, tmp, { max: cap, maxMB: capMB(declared.kind), idleMs });++ if (got.sha !== hash) throw new MediaError(400, 'hash-mismatch');+ const sn = sniffMedia(got.head);+ if (!sn || sn.category !== declared.category) throw new MediaError(415, 'media-type');+ if (got.size > capOf(sn.kind)) throw new MediaError(413, 'media-too-large', { maxMB: capMB(sn.kind) });+ if (sn.ext === 'mp4' || sn.ext === 'mov') {+ const fd = fs.openSync(tmp, 'r');+ let info;+ try { info = mp4Info(fd, got.size); } finally { fs.closeSync(fd); }+ if (!info) throw new MediaError(415, 'media-invalid');+ // One second of slack: the client measures with the platform's player, which rounds.+ if (info.durationSec != null && info.durationSec > L.videoSec + 1) throw new MediaError(413, 'media-too-long', { maxSec: L.videoSec });+ }++ // The same bytes from another device may have landed while these were streaming. The+ // rename and the cache update below are synchronous, so this check cannot be overtaken.+ const now2 = e.hashes.get(hash);+ if (now2) { touch(e, hash); return existed(hash, now2); }+ fs.renameSync(tmp, path.join(e.dir, `${hash}.${sn.ext}`));+ tmp = null;+ e.hashes.set(hash, { ext: sn.ext, size: got.size });+ e.bytes += got.size;+ e.count++;+ // Unreferenced until the state push that names it arrives; the mark is what lets the+ // grace protect it until then, and what lets it go if that push never comes.+ try { markIfUnreferenced(uid, e, hash); } catch (err) { log.error('media: could not mark', e.id, hash, err.message); }+ return { status: 201, body: { ok: true, hash, mime: sn.mime, size: got.size, existed: false } };+ } catch (err) {+ if (err instanceof MediaError) drain(req, 2 * cap);+ throw err;+ } finally {+ if (tmp) { try { fs.unlinkSync(tmp); } catch { /* never created, or already gone */ } }+ e.reserved -= reserved;+ e.inflight--;+ }+ }++ return {+ limits: L,+ usage: uid => usageOf(entry(uid)),+ has: (uid, hash) => entry(uid).hashes.has(hash),+ /** { path, ext, mime, size } of a stored file, or null. Checks the disk, so a file that+ * vanished from under the cache is forgotten rather than answered with a 500. */+ file(uid, hash) {+ const e = entry(uid);+ const f = e.hashes.get(hash);+ if (!f) return null;+ const p = path.join(e.dir, `${hash}.${f.ext}`);+ let st;+ try { st = fs.statSync(p); } catch { forget(e, hash); return null; }+ if (!st.isFile()) return null;+ if (st.size !== f.size) { e.bytes += st.size - f.size; f.size = st.size; }+ return { path: p, ext: f.ext, mime: EXT_MIME[f.ext], size: st.size };+ },+ receive,+ missing(uid, hashes) {+ const e = entry(uid);+ return { missing: [...new Set(hashes)].filter(h => !e.hashes.has(h)), usage: usageOf(e) };+ },+ noteState,+ sweep,+ /** The hourly pass. Only profiles in `uids` are swept, and only when their state reads;+ * a folder that belongs to nobody in db.json is left alone and said so, once per process —+ * only an admin deleting the profile removes a folder. */+ sweepAll({ uids = [], graceMs } = {}) {+ const known = new Map();+ for (const u of uids) { try { known.set(safe(u), u); } catch { /* unusable id */ } }+ const out = { swept: 0, removed: 0, freedBytes: 0, skipped: 0, orphans: 0, tmp: 0 };+ for (const name of userDirs()) {+ out.tmp += cleanTmpOf(path.join(dir, name), HOUR);+ const uid = known.get(name);+ if (uid === undefined) {+ out.orphans++;+ if (!orphansLogged.has(name)) {+ orphansLogged.add(name);+ log.warn(`media: uploads/${name} belongs to no profile in db.json — left alone`);+ }+ continue;+ }+ try {+ const r = sweep(uid, graceMs === undefined ? {} : { graceMs });+ if (r.skipped) out.skipped++; else out.swept++;+ out.removed += r.removed;+ out.freedBytes += r.freedBytes;+ } catch (err) { log.error('media: sweep failed for', name, err.message); }+ }+ return out;+ },+ /** Admin delete: the profile's whole folder. */+ removeUser(uid) {+ const id = safe(uid);+ users.delete(id);+ fs.rmSync(path.join(dir, id), { recursive: true, force: true });+ },+ /** Boot: nothing is in flight yet, so every temp file is a leftover. */+ cleanTmp() {+ let n = 0;+ for (const name of userDirs()) n += cleanTmpOf(path.join(dir, name), 0);+ return n;+ }+ };+}diff --git a/api/test/fixtures/media-refs.json b/api/test/fixtures/media-refs.jsonnew file mode 100644index 000000000..f1b673b36--- /dev/null+++ b/api/test/fixtures/media-refs.json@@ -0,0 +1,430 @@+{+ "about": "Which blob hashes a state references. api/media.js referencedHashes() and frontend/src/lib/media-refs.js referencedHashes() must both give `expect` (compared as sorted lists) for every `state` here: the server keeps and the client keeps exactly this set. Each of customEx[].media.hash and customEx[].media.poster.hash counts on its own when it matches /^[0-9a-f]{64}$/ — deliberately looser than normalizeMediaRef, because this set decides what is kept.",+ "referencedHashes": [+ {+ "name": "no state at all",+ "state": null,+ "expect": []+ },+ {+ "name": "a state without customEx",+ "state": {+ "unit": "kg"+ },+ "expect": []+ },+ {+ "name": "customEx that is not a list",+ "state": {+ "customEx": {+ "x": {+ "id": "cx1",+ "n": "Sandbag carry cx1",+ "bp": "back",+ "custom": true,+ "media": {+ "kind": "image",+ "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "mime": "image/webp",+ "size": 1000,+ "width": 800,+ "height": 600,+ "at": 1+ }+ }+ }+ },+ "expect": []+ },+ {+ "name": "entries that are not exercises are skipped",+ "state": {+ "customEx": [+ null,+ 3,+ "cx",+ [],+ {+ "id": "cx1",+ "n": "Sandbag carry cx1",+ "bp": "back",+ "custom": true,+ "media": {+ "kind": "image",+ "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "mime": "image/webp",+ "size": 1000,+ "width": 800,+ "height": 600,+ "at": 1+ }+ }+ ]+ },+ "expect": [+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ ]+ },+ {+ "name": "an exercise without media",+ "state": {+ "customEx": [+ {+ "id": "cx1",+ "n": "Sandbag carry cx1",+ "bp": "back",+ "custom": true+ }+ ]+ },+ "expect": []+ },+ {+ "name": "a photo and its poster",+ "state": {+ "customEx": [+ {+ "id": "cx1",+ "n": "Sandbag carry cx1",+ "bp": "back",+ "custom": true,+ "media": {+ "kind": "image",+ "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "mime": "image/webp",+ "size": 1000,+ "width": 800,+ "height": 600,+ "poster": {+ "hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+ "mime": "image/webp",+ "size": 200,+ "width": 480,+ "height": 360+ },+ "at": 1+ }+ }+ ]+ },+ "expect": [+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"+ ]+ },+ {+ "name": "a video without a poster",+ "state": {+ "customEx": [+ {+ "id": "cx1",+ "n": "Sandbag carry cx1",+ "bp": "back",+ "custom": true,+ "media": {+ "kind": "video",+ "hash": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",+ "mime": "video/mp4",+ "size": 5000000,+ "width": 1920,+ "height": 1080,+ "dur": 12.4,+ "codec": "avc1",+ "at": 1+ }+ }+ ]+ },+ "expect": [+ "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"+ ]+ },+ {+ "name": "the same file on two exercises counts once",+ "state": {+ "customEx": [+ {+ "id": "cx1",+ "n": "Sandbag carry cx1",+ "bp": "back",+ "custom": true,+ "media": {+ "kind": "image",+ "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "mime": "image/webp",+ "size": 1000,+ "width": 800,+ "height": 600,+ "poster": {+ "hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+ "mime": "image/webp",+ "size": 200,+ "width": 480,+ "height": 360+ },+ "at": 1+ }+ },+ {+ "id": "cx2",+ "n": "Sandbag carry cx2",+ "bp": "back",+ "custom": true,+ "media": {+ "kind": "image",+ "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "mime": "image/webp",+ "size": 1000,+ "width": 800,+ "height": 600,+ "poster": {+ "hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+ "mime": "image/webp",+ "size": 200,+ "width": 480,+ "height": 360+ },+ "at": 1+ }+ }+ ]+ },+ "expect": [+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"+ ]+ },+ {+ "name": "an upper-case hash is not a hash",+ "state": {+ "customEx": [+ {+ "id": "cx1",+ "n": "Sandbag carry cx1",+ "bp": "back",+ "custom": true,+ "media": {+ "kind": "image",+ "hash": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",+ "mime": "image/webp",+ "size": 1000,+ "width": 800,+ "height": 600,+ "at": 1+ }+ }+ ]+ },+ "expect": []+ },+ {+ "name": "a hash one character short is not a hash",+ "state": {+ "customEx": [+ {+ "id": "cx1",+ "n": "Sandbag carry cx1",+ "bp": "back",+ "custom": true,+ "media": {+ "kind": "image",+ "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "mime": "image/webp",+ "size": 1000,+ "width": 800,+ "height": 600,+ "at": 1+ }+ }+ ]+ },+ "expect": []+ },+ {+ "name": "a hash with a path in it is not a hash",+ "state": {+ "customEx": [+ {+ "id": "cx1",+ "n": "Sandbag carry cx1",+ "bp": "back",+ "custom": true,+ "media": {+ "kind": "image",+ "hash": "../aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "mime": "image/webp",+ "size": 1000,+ "width": 800,+ "height": 600,+ "at": 1+ }+ }+ ]+ },+ "expect": []+ },+ {+ "name": "a hash that is not a string",+ "state": {+ "customEx": [+ {+ "id": "cx1",+ "n": "Sandbag carry cx1",+ "bp": "back",+ "custom": true,+ "media": {+ "kind": "image",+ "hash": 12345,+ "mime": "image/webp",+ "size": 1000,+ "width": 800,+ "height": 600,+ "at": 1+ }+ }+ ]+ },+ "expect": []+ },+ {+ "name": "media that is not an object",+ "state": {+ "customEx": [+ {+ "id": "cx1",+ "n": "Sandbag carry cx1",+ "bp": "back",+ "custom": true,+ "media": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ },+ {+ "id": "cx2",+ "n": "Sandbag carry cx2",+ "bp": "back",+ "custom": true,+ "media": [+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"+ ]+ }+ ]+ },+ "expect": []+ },+ {+ "name": "each field counts on its own: a good poster under a malformed main hash is kept",+ "state": {+ "customEx": [+ {+ "id": "cx1",+ "n": "Sandbag carry cx1",+ "bp": "back",+ "custom": true,+ "media": {+ "kind": "image",+ "hash": "nope",+ "mime": "image/webp",+ "size": 1000,+ "width": 800,+ "height": 600,+ "poster": {+ "hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+ "mime": "image/webp",+ "size": 200,+ "width": 480,+ "height": 360+ },+ "at": 1+ }+ }+ ]+ },+ "expect": [+ "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"+ ]+ },+ {+ "name": "a ref the client would not render (unknown mime) still keeps its file",+ "state": {+ "customEx": [+ {+ "id": "cx1",+ "n": "Sandbag carry cx1",+ "bp": "back",+ "custom": true,+ "media": {+ "kind": "image",+ "hash": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",+ "mime": "image/svg+xml",+ "size": 1000,+ "width": 800,+ "height": 600,+ "at": 1+ }+ }+ ]+ },+ "expect": [+ "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ ]+ },+ {+ "name": "stray img/gif fields from a fork build are not references",+ "state": {+ "customEx": [+ {+ "id": "cx1",+ "n": "Sandbag carry cx1",+ "bp": "back",+ "custom": true,+ "img": "http://x/y.jpg",+ "gif": "data:image/gif;base64,R0lG"+ }+ ]+ },+ "expect": []+ },+ {+ "name": "only customEx is a source: media-like fields elsewhere do not count",+ "state": {+ "routines": [+ {+ "id": "r1",+ "ex": [+ {+ "id": "cx1",+ "media": {+ "kind": "image",+ "hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ "mime": "image/webp",+ "size": 1000,+ "width": 800,+ "height": 600,+ "at": 1+ }+ }+ ]+ }+ ],+ "workouts": [+ {+ "entries": [+ {+ "media": {+ "kind": "image",+ "hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",+ "mime": "image/webp",+ "size": 1000,+ "width": 800,+ "height": 600,+ "at": 1+ }+ }+ ]+ }+ ],+ "customEx": []+ },+ "expect": []+ }+ ]+}diff --git a/api/test/media-gc.test.js b/api/test/media-gc.test.jsnew file mode 100644index 000000000..5e197eff9--- /dev/null+++ b/api/test/media-gc.test.js@@ -0,0 +1,249 @@+/* Which stored photos and videos go, and — the half that matters more — which never do.+ A file goes when the profile's own readable state has not referenced it for the grace period+ (14 days; one hour when the quota is full; none for an explicit sweep). Nothing is ever+ deleted because something is missing or unreadable: the state file, the user in db.json, or+ .gc.json. Other devices and stashes may still hold refs that are gone from the server state,+ and the grace is what keeps their files for them. In-process, with the clock injected. */+import { test } from 'node:test';+import assert from 'node:assert/strict';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import { createMediaStore, mediaLimits, MediaError } from '../media.js';+import * as M from './media-samples.mjs';++const DAY = 86400000, HOUR = 3600000;+const quiet = { warn() {}, error() {}, log() {} };++function setup(t, { limits = {}, states = {} } = {}) {+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'media-gc-'));+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ const dir = path.join(root, 'uploads');+ const clock = { t: 1_800_000_000_000 };+ const S = new Map(Object.entries(states));+ const warnings = [];+ const store = createMediaStore({+ dir, limits: { ...mediaLimits({}), ...limits }, now: () => clock.t,+ readState: uid => {+ const v = S.get(uid);+ if (v instanceof Error) throw v;+ return v ?? null;+ },+ log: { ...quiet, warn: m => warnings.push(m) }+ });+ // Puts a file in place the way an upload leaves it, before the store has looked.+ const place = (uid, bytes, ext = 'jpg') => {+ const d = path.join(dir, uid);+ fs.mkdirSync(d, { recursive: true, mode: 0o700 });+ const h = M.sha(bytes);+ fs.writeFileSync(path.join(d, `${h}.${ext}`), bytes, { mode: 0o600 });+ return h;+ };+ const files = uid => { try { return fs.readdirSync(path.join(dir, uid)).filter(n => !n.startsWith('.')).sort(); } catch { return []; } };+ const marks = uid => JSON.parse(fs.readFileSync(path.join(dir, uid, '.gc.json'), 'utf8'));+ return { dir, clock, S, store, place, files, marks, warnings };+}+const ref = (...hashes) => ({ customEx: hashes.map((h, i) => ({ id: 'cx' + i, n: 'x', bp: 'back', custom: true, media: { kind: 'image', hash: h, mime: 'image/jpeg', size: 1, width: 1, height: 1, at: 1 } })) });++test('noteState marks a file the state stopped referencing and clears the mark when it is back', t => {+ const h = setup(t);+ const a = h.place('u1', M.jpeg()), b = h.place('u1', M.jpeg());+ assert.equal(h.store.noteState('u1', ref(a, b)), false, 'both referenced: nothing to write');+ assert.equal(fs.existsSync(path.join(h.dir, 'u1', '.gc.json')), false);++ assert.equal(h.store.noteState('u1', ref(a)), true);+ assert.deepEqual(h.marks('u1'), { [b]: h.clock.t });++ h.clock.t += DAY;+ assert.equal(h.store.noteState('u1', ref(a)), false, 'an existing mark keeps its time');+ assert.deepEqual(h.marks('u1'), { [b]: h.clock.t - DAY });++ assert.equal(h.store.noteState('u1', ref(a, b)), true);+ assert.deepEqual(h.marks('u1'), {});+});++test('noteState does nothing at all for a profile that never uploaded', t => {+ const h = setup(t);+ assert.equal(h.store.noteState('u1', ref('a'.repeat(64))), false);+ assert.equal(fs.existsSync(h.dir), false, 'no uploads folder is created by a state push');+});++test('the hourly sweep keeps referenced files and removes the rest only after the grace', t => {+ const h = setup(t);+ const keep = h.place('u1', M.jpeg()), drop = h.place('u1', M.jpeg(3000));+ h.S.set('u1', ref(keep));+ assert.deepEqual(h.store.usage('u1'), { bytes: 4000, count: 2, quotaBytes: 200 * 1048576 });+ h.store.noteState('u1', ref(keep));++ h.clock.t += 14 * DAY - 1;+ assert.deepEqual(h.store.sweep('u1'), { removed: 0, freedBytes: 0, skipped: false });+ assert.equal(h.files('u1').length, 2, 'a day short of the grace: still there');++ h.clock.t += 1;+ assert.deepEqual(h.store.sweep('u1'), { removed: 1, freedBytes: 3000, skipped: false });+ assert.deepEqual(h.files('u1'), [`${keep}.jpg`]);+ assert.deepEqual(h.store.usage('u1'), { bytes: 1000, count: 1, quotaBytes: 200 * 1048576 });+ assert.deepEqual(h.marks('u1'), {}, 'the mark goes with the file');+});++test('a file nobody marked yet starts its grace at the first sweep that sees it unreferenced', t => {+ const h = setup(t);+ const x = h.place('u1', M.jpeg());+ h.S.set('u1', ref());+ assert.equal(h.store.sweep('u1').removed, 0);+ assert.deepEqual(h.marks('u1'), { [x]: h.clock.t });+ h.clock.t += 14 * DAY;+ assert.equal(h.store.sweep('u1').removed, 1);+});++test('an explicit sweep (grace 0) removes every unreferenced file at once', t => {+ const h = setup(t);+ const keep = h.place('u1', M.jpeg()), a = h.place('u1', M.png()), b = h.place('u1', M.gif(), 'gif');+ h.S.set('u1', ref(keep));+ const r = h.store.sweep('u1', { graceMs: 0 });+ assert.equal(r.removed, 2);+ assert.deepEqual(h.files('u1'), [`${keep}.jpg`]);+ assert.ok(!h.store.has('u1', a) && !h.store.has('u1', b));+});++test('a profile whose state cannot be read is never swept — missing, unparsable or throwing', t => {+ const h = setup(t);+ for (const [uid, state] of [['gone', undefined], ['bad', null], ['arr', []], ['boom', new Error('EIO')]]) {+ h.place(uid, M.jpeg());+ if (state !== undefined) h.S.set(uid, state);+ h.clock.t += 30 * DAY;+ const r = h.store.sweep(uid, { graceMs: 0 });+ assert.deepEqual(r, { removed: 0, freedBytes: 0, skipped: true }, uid);+ assert.equal(h.files(uid).length, 1, uid);+ }+});++test('an unreadable .gc.json is no marks: the grace starts over and nothing is deleted for it', t => {+ const h = setup(t);+ const x = h.place('u1', M.jpeg());+ h.S.set('u1', ref());+ h.store.sweep('u1'); // marked now+ h.clock.t += 20 * DAY;+ fs.writeFileSync(path.join(h.dir, 'u1', '.gc.json'), '{"torn');+ assert.equal(h.store.sweep('u1').removed, 0, 'a mark that cannot be read is not a mark');+ assert.deepEqual(h.marks('u1'), { [x]: h.clock.t }, 'a fresh mark replaces the torn file');+ h.clock.t += 14 * DAY;+ assert.equal(h.store.sweep('u1').removed, 1);+});++test('.gc.json turned into a directory: the hourly sweep deletes nothing, noteState throws for the caller to catch', t => {+ const h = setup(t);+ h.place('u1', M.jpeg());+ h.S.set('u1', ref());+ fs.mkdirSync(path.join(h.dir, 'u1', '.gc.json'));+ h.clock.t += 60 * DAY;+ assert.equal(h.store.sweep('u1').removed, 0);+ assert.equal(h.files('u1').length, 1);+ assert.throws(() => h.store.noteState('u1', { customEx: [] }));+ assert.equal(fs.existsSync(path.join(h.dir, 'u1', '.gc.json.tmp')), false, 'the failed write leaves no temp file');+});++test('sweepAll with an empty db.users removes no folder and no file, and says so once', t => {+ const h = setup(t);+ h.place('u1', M.jpeg());+ h.place('u2', M.jpeg());+ h.S.set('u1', ref());+ h.S.set('u2', ref());+ h.clock.t += 60 * DAY;+ for (let i = 0; i < 3; i++) {+ const r = h.store.sweepAll({ uids: [], graceMs: 0 });+ assert.equal(r.removed, 0);+ assert.equal(r.orphans, 2);+ }+ assert.equal(h.files('u1').length + h.files('u2').length, 2);+ assert.equal(h.warnings.length, 2, 'one line per folder per process, not one per hour');+ assert.match(h.warnings[0], /belongs to no profile/);+});++test('sweepAll sweeps the profiles it is given, and only when their state reads', t => {+ const h = setup(t);+ h.place('u1', M.jpeg());+ h.place('u2', M.jpeg());+ h.S.set('u1', ref()); // readable, references nothing+ // u2: no state at all+ const r = h.store.sweepAll({ uids: ['u1', 'u2'], graceMs: 0 });+ assert.equal(r.removed, 1);+ assert.equal(r.swept, 1);+ assert.equal(r.skipped, 1);+ assert.equal(h.files('u1').length, 0);+ assert.equal(h.files('u2').length, 1);+});++test('temp files: the hourly pass removes those older than an hour, boot removes them all', t => {+ const h = setup(t);+ h.place('u1', M.jpeg());+ const tmp = path.join(h.dir, 'u1', '.tmp');+ fs.mkdirSync(tmp, { recursive: true });+ fs.writeFileSync(path.join(tmp, 'old'), 'x');+ fs.writeFileSync(path.join(tmp, 'new'), 'x');+ const old = new Date(h.clock.t - 2 * HOUR), fresh = new Date(h.clock.t - 10 * 60000);+ fs.utimesSync(path.join(tmp, 'old'), old, old);+ fs.utimesSync(path.join(tmp, 'new'), fresh, fresh);+ const r = h.store.sweepAll({ uids: [] });+ assert.equal(r.tmp, 1);+ assert.deepEqual(fs.readdirSync(tmp), ['new']);+ assert.equal(h.store.cleanTmp(), 1);+ assert.deepEqual(fs.readdirSync(tmp), []);+});++test('dotfiles and unknown names are never blobs', t => {+ const h = setup(t);+ const x = h.place('u1', M.jpeg());+ const d = path.join(h.dir, 'u1');+ fs.writeFileSync(path.join(d, '.gc.json'), '{}');+ fs.writeFileSync(path.join(d, `${x}.svg`), '<svg/>');+ fs.writeFileSync(path.join(d, 'notes.txt'), 'hello');+ fs.writeFileSync(path.join(d, `.${'e'.repeat(64)}.jpg`), 'x');+ assert.deepEqual(h.store.usage('u1'), { bytes: 1000, count: 1, quotaBytes: 200 * 1048576 });+ h.S.set('u1', ref());+ h.store.sweep('u1', { graceMs: 0 });+ assert.deepEqual(fs.readdirSync(d).sort(), ['.gc.json', `.${'e'.repeat(64)}.jpg`, '.tmp', `${x}.svg`, 'notes.txt'].filter(n => n !== '.tmp').sort());+});++test('removeUser deletes the whole folder, and an id that sanitises to nothing is refused', t => {+ const h = setup(t);+ h.place('u1', M.jpeg());+ h.place('u2', M.jpeg());+ h.store.removeUser('u1');+ assert.equal(fs.existsSync(path.join(h.dir, 'u1')), false);+ assert.equal(fs.existsSync(path.join(h.dir, 'u2')), true);+ assert.equal(h.store.usage('u1').count, 0);+ // '../..' would otherwise name uploads/ itself — everybody's files.+ assert.throws(() => h.store.removeUser('../..'));+ assert.throws(() => h.store.removeUser(''));+ assert.equal(fs.existsSync(path.join(h.dir, 'u2')), true);+});++test('under quota pressure the grace is one hour', async t => {+ // 2500 bytes of quota; 1000 already stored and unreferenced; a 2000-byte upload needs room.+ const h = setup(t, { limits: { quotaMB: 2500 / 1048576, minFreeMB: 0 } });+ const old = h.place('u1', M.jpeg());+ h.S.set('u1', ref());+ h.store.noteState('u1', ref()); // marked now+ const next = M.jpeg(2000);++ h.clock.t += 30 * 60000; // half an hour: still protected+ await assert.rejects(h.store.receive('u1', M.sha(next), M.fakeReq(next)),+ e => e instanceof MediaError && e.status === 413 && e.code === 'media-quota' && e.extra.quotaMB > 0);+ assert.ok(h.store.has('u1', old));++ h.clock.t += 31 * 60000; // past the hour: it makes room+ const r = await h.store.receive('u1', M.sha(next), M.fakeReq(next));+ assert.equal(r.status, 201);+ assert.ok(!h.store.has('u1', old));+ assert.equal(h.store.usage('u1').bytes, 2000);+});++test('a new upload nobody references yet is marked, so the grace also covers a push that never comes', async t => {+ const h = setup(t, { limits: { minFreeMB: 0 } });+ const a = M.jpeg(), b = M.jpeg();+ h.S.set('u1', ref(M.sha(b)));+ await h.store.receive('u1', M.sha(a), M.fakeReq(a));+ await h.store.receive('u1', M.sha(b), M.fakeReq(b));+ assert.deepEqual(h.marks('u1'), { [M.sha(a)]: h.clock.t }, 'the referenced one is not marked');+});diff --git a/api/test/media-receive.test.js b/api/test/media-receive.test.jsnew file mode 100644index 000000000..fc29d3e7d--- /dev/null+++ b/api/test/media-receive.test.js@@ -0,0 +1,207 @@+/* receive(): one upload from the first byte to the rename. In-process, so the memory it holds+ and the state it leaves behind on every way out can be looked at directly. The production API+ runs with a 128 MB memory limit and a video may be 40 MB: the bytes must stream to disk, never+ collect in memory. And whatever ends an upload — success, refusal, a client hanging up, a+ stall — the temp file goes, and the in-flight slot and the quota reservation come back. */+import { test } from 'node:test';+import assert from 'node:assert/strict';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import v8 from 'node:v8';+import vm from 'node:vm';+import { Readable } from 'node:stream';+import { createMediaStore, mediaLimits, MediaError } from '../media.js';+import * as M from './media-samples.mjs';++v8.setFlagsFromString('--expose-gc');+const gc = vm.runInNewContext('gc');+const quiet = { warn() {}, error() {}, log() {} };++function setup(t, limits = {}, opts = {}) {+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'media-recv-'));+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ const dir = path.join(root, 'uploads');+ const store = createMediaStore({ dir, limits: { ...mediaLimits({}), minFreeMB: 0, ...limits }, log: quiet, ...opts });+ const tmpFiles = uid => { try { return fs.readdirSync(path.join(dir, uid, '.tmp')); } catch { return []; } };+ const files = uid => { try { return fs.readdirSync(path.join(dir, uid)).filter(n => !n.startsWith('.')); } catch { return []; } };+ return { dir, store, tmpFiles, files };+}+const isMediaError = (status, code) => e => e instanceof MediaError && e.status === status && e.code === code;++test('a 30 MB upload streams through with a flat memory profile', async t => {+ const h = setup(t);+ // A real MP4 head so the whole path runs, sniff and box walk included; the rest is fresh+ // 64 KB buffers, each its own ArrayBuffer, so a receiver that kept them would show.+ const headPart = M.mp4({ seconds: 20, mdatBytes: 0 });+ const TOTAL = 30 * 1024 * 1024, CHUNK = 64 * 1024;+ const bodyLen = TOTAL - headPart.length;+ // mdat's size field has to cover what follows it for mp4Info to accept the file.+ headPart.writeUInt32BE(8 + bodyLen, headPart.length - 8);+ const crypto = await import('node:crypto');+ const hasher = crypto.createHash('sha256').update(headPart);+ const lenAt = i => Math.min(CHUNK, bodyLen - i * CHUNK);+ const fillOf = i => (i * 7) & 0xff;+ const chunkAt = i => Buffer.allocUnsafeSlow(lenAt(i)).fill(fillOf(i));+ const nChunks = Math.ceil(bodyLen / CHUNK);+ // Hashed through one scratch buffer: 30 MB of garbage made here would still be waiting for+ // the collector when the baseline is taken, and its release would then hide real growth.+ const scratch = Buffer.alloc(CHUNK);+ for (let i = 0; i < nChunks; i++) hasher.update(scratch.fill(fillOf(i)).subarray(0, lenAt(i)));+ const hash = hasher.digest('hex');++ // ArrayBuffer backing stores are released by a task after the collection, not inside it.+ const settle = async () => { gc(); await new Promise(r => setImmediate(r)); gc(); };+ await settle();+ const base = process.memoryUsage().arrayBuffers;+ let peak = 0;+ async function* body() {+ yield headPart;+ for (let i = 0; i < nChunks; i++) {+ if (i % 32 === 0) { gc(); peak = Math.max(peak, process.memoryUsage().arrayBuffers - base); }+ yield chunkAt(i);+ }+ }+ const req = Readable.from(body(), { objectMode: false });+ req.headers = { 'content-type': 'video/mp4', 'content-length': String(TOTAL) };+ const r = await h.store.receive('u1', hash, req);+ assert.equal(r.status, 201);+ assert.equal(r.body.size, TOTAL);+ await settle();+ peak = Math.max(peak, process.memoryUsage().arrayBuffers - base);+ assert.ok(peak < 8 * 1024 * 1024, `arrayBuffers grew by ${(peak / 1048576).toFixed(1)} MB`);+ assert.equal(fs.statSync(path.join(h.dir, 'u1', `${hash}.mp4`)).size, TOTAL);+});++test('a client that hangs up mid-upload leaves no temp file and gives back its slot and reservation', async t => {+ const h = setup(t, { quotaMB: 1 });+ const bytes = M.jpeg(500000);+ let sent = 0;+ const req = new Readable({+ read() {+ if (sent >= 3) { this.destroy(Object.assign(new Error('aborted'), { code: 'ECONNRESET' })); return; }+ this.push(bytes.subarray(sent * 65536, ++sent * 65536));+ }+ });+ req.headers = { 'content-type': 'image/jpeg', 'content-length': String(bytes.length) };+ await assert.rejects(h.store.receive('u1', M.sha(bytes), req), e => e.clientGone === true);+ assert.deepEqual(h.tmpFiles('u1'), []);+ assert.deepEqual(h.files('u1'), []);+ // The reservation came back: a file that needs almost the whole quota fits.+ const full = M.jpeg(1000000);+ assert.equal((await h.store.receive('u1', M.sha(full), M.fakeReq(full))).status, 201);+ // And the slot: two more at once are both taken.+ const a = M.jpeg(1000), b = M.jpeg(1000);+ const h2 = setup(t);+ const results = await Promise.all([h2.store.receive('u1', M.sha(a), M.fakeReq(a)), h2.store.receive('u1', M.sha(b), M.fakeReq(b))]);+ assert.deepEqual(results.map(r => r.status), [201, 201]);+});++test('an upload that stops sending is cut off after the idle time with 408', async t => {+ const h = setup(t, {}, { idleMs: 80 });+ const bytes = M.jpeg(200000);+ const req = new Readable({ read() {} }); // pushes one chunk, then nothing, ever+ req.push(bytes.subarray(0, 65536));+ req.headers = { 'content-type': 'image/jpeg', 'content-length': String(bytes.length) };+ await assert.rejects(h.store.receive('u1', M.sha(bytes), req), isMediaError(408, 'timeout'));+ assert.deepEqual(h.tmpFiles('u1'), []);+});++test('an upload without a length is capped while streaming, not after', async t => {+ const h = setup(t, { imageMB: 0.1 });+ const bytes = M.jpeg(300000);+ const req = M.fakeReq(bytes, { length: null });+ await assert.rejects(h.store.receive('u1', M.sha(bytes), req), e => isMediaError(413, 'media-too-large')(e) && e.extra.maxMB === 0.1);+ assert.deepEqual(h.tmpFiles('u1'), []);+ assert.deepEqual(h.store.usage('u1'), { bytes: 0, count: 0, quotaBytes: 200 * 1048576 });+});++test('a declared length over the cap is refused before a byte is read', async t => {+ const h = setup(t, { imageMB: 0.1 });+ const bytes = M.jpeg(300000);+ await assert.rejects(h.store.receive('u1', M.sha(bytes), M.fakeReq(bytes)), isMediaError(413, 'media-too-large'));+ assert.equal(fs.existsSync(path.join(h.dir, 'u1', '.tmp')), false, 'no temp file was ever opened');+});++test('the file is stored as what it is: a JPEG declared as a GIF is a JPEG, over the image cap it is refused', async t => {+ const h = setup(t, { imageMB: 0.01, gifMB: 1 });+ const small = M.jpeg(5000);+ const r = await h.store.receive('u1', M.sha(small), M.fakeReq(small, { declare: 'image/gif' }));+ assert.deepEqual(r.body, { ok: true, hash: M.sha(small), mime: 'image/jpeg', size: 5000, existed: false });+ assert.deepEqual(h.files('u1'), [`${M.sha(small)}.jpg`]);+ const big = M.jpeg(50000);+ await assert.rejects(h.store.receive('u1', M.sha(big), M.fakeReq(big, { declare: 'image/gif' })), e => isMediaError(413, 'media-too-large')(e) && e.extra.maxMB === 0.01);+});++test('a video declared as an image, SVG declared as PNG, and an unknown type are refused', async t => {+ const h = setup(t);+ const vid = M.mp4();+ await assert.rejects(h.store.receive('u1', M.sha(vid), M.fakeReq(vid, { declare: 'image/jpeg' })), isMediaError(415, 'media-type'));+ const svg = M.svg();+ await assert.rejects(h.store.receive('u1', M.sha(svg), M.fakeReq(svg, { declare: 'image/png' })), isMediaError(415, 'media-type'));+ await assert.rejects(h.store.receive('u1', M.sha(svg), M.fakeReq(svg, { declare: 'image/svg+xml' })), isMediaError(415, 'media-type'));+ await assert.rejects(h.store.receive('u1', M.sha(svg), M.fakeReq(svg, { declare: 'text/html' })), isMediaError(415, 'media-type'));+ assert.deepEqual(h.files('u1'), []);+ assert.deepEqual(h.tmpFiles('u1'), []);+});++test('bytes that do not hash to their name are refused and leave nothing', async t => {+ const h = setup(t);+ const bytes = M.jpeg();+ await assert.rejects(h.store.receive('u1', 'f'.repeat(64), M.fakeReq(bytes)), isMediaError(400, 'hash-mismatch'));+ assert.deepEqual(h.files('u1'), []);+ assert.deepEqual(h.tmpFiles('u1'), []);+});++test('videos: too long, unreadable, and a WebM that only the byte cap applies to', async t => {+ const h = setup(t, { videoSec: 60 });+ const ok = M.mp4({ seconds: 61 }); // one second of slack for rounding+ assert.equal((await h.store.receive('u1', M.sha(ok), M.fakeReq(ok, { declare: 'video/mp4' }))).status, 201);+ const long = M.mp4({ seconds: 62 });+ await assert.rejects(h.store.receive('u1', M.sha(long), M.fakeReq(long, { declare: 'video/mp4' })), e => isMediaError(413, 'media-too-long')(e) && e.extra.maxSec === 60);+ const broken = Buffer.concat([M.ftyp(), M.box('mdat', Buffer.alloc(100))]);+ await assert.rejects(h.store.receive('u1', M.sha(broken), M.fakeReq(broken, { declare: 'video/mp4' })), isMediaError(415, 'media-invalid'));+ const mov = M.mp4({ brand: 'qt ', seconds: 10 });+ const r = await h.store.receive('u1', M.sha(mov), M.fakeReq(mov, { declare: 'video/mp4' }));+ assert.equal(r.body.mime, 'video/quicktime');+ const webm = M.ebml('webm', 5000);+ assert.equal((await h.store.receive('u1', M.sha(webm), M.fakeReq(webm, { declare: 'video/webm' }))).body.mime, 'video/webm');+ assert.deepEqual(h.files('u1').map(n => n.split('.')[1]).sort(), ['mov', 'mp4', 'webm']);+});++test('a file the server already has is answered as such and counted once', async t => {+ const h = setup(t);+ const bytes = M.png(4000);+ const first = await h.store.receive('u1', M.sha(bytes), M.fakeReq(bytes, { declare: 'image/png' }));+ const again = await h.store.receive('u1', M.sha(bytes), M.fakeReq(bytes, { declare: 'image/png' }));+ assert.equal(first.status, 201);+ assert.deepEqual([again.status, again.body], [200, { ok: true, hash: M.sha(bytes), mime: 'image/png', size: 4000, existed: true }]);+ assert.deepEqual(h.store.usage('u1'), { bytes: 4000, count: 1, quotaBytes: 200 * 1048576 });+ // The same bytes from two devices at once: one file, counted once.+ const both = M.jpeg(300000);+ const rs = await Promise.all([1, 2].map(() => h.store.receive('u2', M.sha(both), M.fakeReq(both, { chunk: 4096 }))));+ assert.deepEqual(rs.map(r => r.status).sort(), [200, 201]);+ assert.deepEqual(h.store.usage('u2'), { bytes: 300000, count: 1, quotaBytes: 200 * 1048576 });+ assert.deepEqual(h.tmpFiles('u2'), []);+});++test('a third upload at once is refused as busy', async t => {+ const h = setup(t);+ const stalled = () => { const r = new Readable({ read() {} }); r.headers = { 'content-type': 'image/jpeg', 'content-length': '100000' }; return r; };+ const a = stalled(), b = stalled();+ const pa = h.store.receive('u1', 'a'.repeat(64), a), pb = h.store.receive('u1', 'b'.repeat(64), b);+ const c = M.jpeg();+ await assert.rejects(h.store.receive('u1', M.sha(c), M.fakeReq(c)), e => isMediaError(429, 'busy')(e) && e.extra.retryAfter === 5 && e.headers['Retry-After'] === '5');+ // Another profile is not affected.+ assert.equal((await h.store.receive('u2', M.sha(c), M.fakeReq(c))).status, 201);+ a.destroy(new Error('aborted')); b.destroy(new Error('aborted'));+ await Promise.allSettled([pa, pb]);+ assert.equal((await h.store.receive('u1', M.sha(c), M.fakeReq(c))).status, 201, 'the slots came back');+});++test('the disk floor refuses an upload with 507 and holds no reservation afterwards', async t => {+ const h = setup(t, { minFreeMB: 1e9 });+ const bytes = M.jpeg();+ await assert.rejects(h.store.receive('u1', M.sha(bytes), M.fakeReq(bytes)), isMediaError(507, 'storage-full'));+ assert.deepEqual(h.tmpFiles('u1'), []);+});diff --git a/api/test/media-refs.test.js b/api/test/media-refs.test.jsnew file mode 100644index 000000000..22079822c--- /dev/null+++ b/api/test/media-refs.test.js@@ -0,0 +1,31 @@+/* referencedHashes() decides which stored files the server keeps. The app has its own copy+ (frontend/src/lib/media-refs.js) deciding which local files the device keeps, and the two must+ never disagree: a hash the server counts and the device does not is a file one side deletes+ while the other still shows it. Both are tested against this one fixture. */+import { test } from 'node:test';+import assert from 'node:assert/strict';+import fs from 'node:fs';+import path from 'node:path';+import { fileURLToPath } from 'node:url';+import { referencedHashes } from '../media.js';++const FIXTURE = path.join(path.dirname(fileURLToPath(import.meta.url)), 'fixtures', 'media-refs.json');+const { referencedHashes: cases } = JSON.parse(fs.readFileSync(FIXTURE, 'utf8'));++test('the shared fixture is there and has cases', () => {+ assert.ok(Array.isArray(cases) && cases.length >= 10);+});++for (const c of cases) {+ test(`referencedHashes: ${c.name}`, () => {+ const got = referencedHashes(c.state);+ assert.ok(got instanceof Set);+ assert.deepEqual([...got].sort(), [...c.expect].sort());+ });+}++test('never throws, whatever the state is', () => {+ for (const S of [undefined, 0, '', 'x', [], [1], { customEx: null }, { customEx: [{ media: null }] }, { customEx: [{ media: { poster: 'x' } }] }]) {+ assert.deepEqual([...referencedHashes(S)], []);+ }+});diff --git a/api/test/media-samples.mjs b/api/test/media-samples.mjsnew file mode 100644index 000000000..225643dcd--- /dev/null+++ b/api/test/media-samples.mjs@@ -0,0 +1,99 @@+/* Tiny synthetic media files for the media tests: just enough structure for the magic bytes and+ the MP4 box walk, built in code so the repo carries no binary fixtures and every test states+ exactly which bytes it feeds in. Not a *.test.js file, so `npm test` does not run it. */+import crypto from 'node:crypto';+import { Readable } from 'node:stream';++export const sha = buf => crypto.createHash('sha256').update(buf).digest('hex');+const pad = n => crypto.randomBytes(Math.max(0, n));++export const jpeg = (n = 1000) => Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), pad(n - 4)]);+export const png = (n = 1000) => Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), pad(n - 8)]);+export const gif = (n = 1000, v = '89a') => Buffer.concat([Buffer.from('GIF' + v, 'latin1'), pad(n - 6)]);+export function webp(n = 1000) {+ const b = Buffer.concat([Buffer.from('RIFF', 'latin1'), Buffer.alloc(4), Buffer.from('WEBPVP8 ', 'latin1'), pad(n - 16)]);+ b.writeUInt32LE(b.length - 8, 4);+ return b;+}+export const svg = () => Buffer.from('<?xml version="1.0"?><svg xmlns="http://www.w3.org/2000/svg" onload="alert(1)"></svg>');+export const html = () => Buffer.from('<!doctype html><html><script>alert(1)</script></html>');+export const pdf = () => Buffer.from('%PDF-1.7\n%âãÏÓ\n1 0 obj<<>>endobj\n', 'latin1');+export const zip = () => Buffer.concat([Buffer.from([0x50, 0x4b, 0x03, 0x04]), pad(100)]);++/* ---- EBML (WebM / Matroska) ---- */+export function ebml(docType = 'webm', n = 400) {+ const el = (id, body) => Buffer.concat([Buffer.from(id), Buffer.from([0x80 | body.length]), body]);+ const kids = Buffer.concat([+ el([0x42, 0x86], Buffer.from([1])), el([0x42, 0xf7], Buffer.from([1])),+ el([0x42, 0xf2], Buffer.from([4])), el([0x42, 0xf3], Buffer.from([8])),+ el([0x42, 0x82], Buffer.from(docType, 'latin1')),+ el([0x42, 0x87], Buffer.from([4])), el([0x42, 0x85], Buffer.from([2]))+ ]);+ const head = Buffer.concat([Buffer.from([0x1a, 0x45, 0xdf, 0xa3, 0x80 | kids.length]), kids]);+ return Buffer.concat([head, pad(n - head.length)]);+}++/* ---- ISO-BMFF (MP4 / MOV / HEIC / M4A) ---- */+export function box(type, ...parts) {+ const body = Buffer.concat(parts.map(p => (Buffer.isBuffer(p) ? p : Buffer.from(p))));+ const h = Buffer.alloc(8);+ h.writeUInt32BE(8 + body.length, 0);+ h.write(type, 4, 'latin1');+ return Buffer.concat([h, body]);+}+// size = 1 and a 64-bit size after the type, which is how a camera writes an mdat over 4 GB.+export function box64(type, body) {+ const h = Buffer.alloc(16);+ h.writeUInt32BE(1, 0);+ h.write(type, 4, 'latin1');+ h.writeBigUInt64BE(BigInt(16 + body.length), 8);+ return Buffer.concat([h, body]);+}+export const ftyp = (brand = 'isom', compat = ['isom', 'mp41']) =>+ box('ftyp', Buffer.from(brand, 'latin1'), Buffer.alloc(4), Buffer.from(compat.join(''), 'latin1'));++export function mvhd({ version = 0, timescale = 1000, duration = 5000 } = {}) {+ if (version === 1) {+ const b = Buffer.alloc(112);+ b[0] = 1;+ b.writeUInt32BE(timescale, 20);+ b.writeBigUInt64BE(BigInt(duration), 24);+ return box('mvhd', b);+ }+ const b = Buffer.alloc(100);+ b.writeUInt32BE(timescale, 12);+ b.writeUInt32BE(duration >>> 0, 16);+ return box('mvhd', b);+}+export function mehd(duration) {+ const b = Buffer.alloc(8);+ b.writeUInt32BE(duration, 4);+ return box('mehd', b);+}++/**+ * A structurally valid MP4/MOV: ftyp, then moov (mvhd + an empty trak) and mdat in either order.+ * `seconds` is written as mvhd's duration at `timescale`; `moovPad` inflates moov with a `free`+ * box, `mdatBytes` sizes the media data.+ */+export function mp4({ brand = 'isom', seconds = 5, timescale = 1000, version = 0, moovAtEnd = false, mdatBytes = 2000, moovPad = 0, extraMoov = [], bigMdat = false } = {}) {+ const moov = box('moov', mvhd({ version, timescale, duration: Math.round(seconds * timescale) }),+ box('trak', box('tkhd', Buffer.alloc(84))), ...extraMoov, ...(moovPad ? [box('free', Buffer.alloc(moovPad))] : []));+ const mdat = bigMdat ? box64('mdat', pad(mdatBytes)) : box('mdat', pad(mdatBytes));+ return Buffer.concat(moovAtEnd ? [ftyp(brand), mdat, moov] : [ftyp(brand), moov, mdat]);+}++/* ---- a request stand-in for receive() ---- */+/**+ * What receive() reads off a request: `headers` and a byte stream. `bytes` is sent in `chunk`+ * pieces; `declare` is the Content-Type, `length` the Content-Length (true = the real length,+ * a number = that claim, null = none, as with chunked encoding).+ */+export function fakeReq(bytes, { declare = 'image/jpeg', length = true, chunk = 64 * 1024 } = {}) {+ const parts = [];+ for (let i = 0; i < bytes.length; i += chunk) parts.push(bytes.subarray(i, i + chunk));+ const req = Readable.from(parts, { objectMode: false });+ req.headers = { 'content-type': declare };+ if (length !== null) req.headers['content-length'] = String(length === true ? bytes.length : length);+ return req;+}diff --git a/api/test/media-sniff.test.js b/api/test/media-sniff.test.jsnew file mode 100644index 000000000..15f12e71c--- /dev/null+++ b/api/test/media-sniff.test.js@@ -0,0 +1,112 @@+/* What the server stores is decided by the first bytes of a file and nothing else: not its name,+ not the Content-Type the client declared. Seven types pass; SVG and HTML — the two that would+ run in the app's origin if a browser were ever talked into rendering them — and everything+ else do not. mp4Info is the one place the server looks deeper, to read a video's length off+ its headers without loading it. */+import { test } from 'node:test';+import assert from 'node:assert/strict';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import { sniffMedia, mp4Info, MEDIA_TYPES } from '../media.js';+import * as M from './media-samples.mjs';++const head = b => b.subarray(0, 64);+const kindOf = b => sniffMedia(head(b));++test('each of the seven stored types is recognised from its magic bytes', () => {+ const cases = [+ [M.jpeg(), 'image/jpeg', 'image', 'still', 'jpg'],+ [M.png(), 'image/png', 'image', 'still', 'png'],+ [M.webp(), 'image/webp', 'image', 'still', 'webp'],+ [M.gif(), 'image/gif', 'gif', 'still', 'gif'],+ [M.gif(1000, '87a'), 'image/gif', 'gif', 'still', 'gif'],+ [M.mp4(), 'video/mp4', 'video', 'video', 'mp4'],+ [M.mp4({ brand: 'qt ' }), 'video/quicktime', 'video', 'video', 'mov'],+ [M.ebml('webm'), 'video/webm', 'video', 'video', 'webm']+ ];+ for (const [bytes, mime, kind, category, ext] of cases) {+ assert.deepEqual(kindOf(bytes), { mime, kind, category, ext }, mime);+ }+ assert.equal(Object.keys(MEDIA_TYPES).length, 7);+});++test('every MP4 brand on the list is a video, and a brand nobody vouched for is not', () => {+ for (const brand of ['isom', 'iso2', 'iso3', 'iso4', 'iso5', 'iso6', 'mp41', 'mp42', 'avc1', 'M4V ', 'M4VP', 'dash', 'mmp4', 'MSNV', '3gp4', '3gp5', '3gp6', '3g2a']) {+ assert.equal(kindOf(M.mp4({ brand }))?.mime, 'video/mp4', brand);+ }+ for (const brand of ['abcd', 'iso9', 'mp71', 'crx ']) assert.equal(kindOf(M.mp4({ brand })), null, brand);+});++test('HEIC and AVIF stills are refused — the app re-encodes them, it never uploads them', () => {+ for (const brand of ['heic', 'heix', 'heim', 'heis', 'hevc', 'mif1', 'msf1', 'avif', 'avis']) {+ assert.equal(kindOf(M.mp4({ brand })), null, brand);+ }+});++test('SVG, HTML, PDF, ZIP, M4A audio and Matroska are refused', () => {+ assert.equal(kindOf(M.svg()), null, 'svg');+ assert.equal(kindOf(M.html()), null, 'html');+ assert.equal(kindOf(M.pdf()), null, 'pdf');+ assert.equal(kindOf(M.zip()), null, 'zip');+ assert.equal(kindOf(M.mp4({ brand: 'M4A ' })), null, 'm4a');+ assert.equal(kindOf(M.ebml('matroska')), null, 'matroska');+ assert.equal(kindOf(Buffer.from('BM' + 'x'.repeat(60))), null, 'bmp');+ assert.equal(kindOf(Buffer.from([0x49, 0x49, 0x2a, 0x00, 1, 2, 3, 4])), null, 'tiff');+ assert.equal(kindOf(Buffer.from([0, 0, 1, 0, 1, 0])), null, 'ico');+ assert.equal(kindOf(Buffer.alloc(0)), null, 'empty');+ assert.equal(kindOf(Buffer.from([0xff, 0xd8])), null, 'truncated jpeg');+});++/* ---------------- mp4Info ---------------- */++const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'media-sniff-'));+test.after(() => fs.rmSync(dir, { recursive: true, force: true }));+let n = 0;+function info(bytes) {+ const f = path.join(dir, `f${n++}.mp4`);+ fs.writeFileSync(f, bytes);+ const fd = fs.openSync(f, 'r');+ try { return mp4Info(fd, bytes.length); } finally { fs.closeSync(fd); }+}++test('mp4Info reads the length from mvhd, version 0 and version 1', () => {+ assert.deepEqual(info(M.mp4({ seconds: 12.5 })), { brand: 'isom', durationSec: 12.5 });+ assert.deepEqual(info(M.mp4({ version: 1, seconds: 42, timescale: 90000 })), { brand: 'isom', durationSec: 42 });+ assert.deepEqual(info(M.mp4({ brand: 'qt ', seconds: 61, timescale: 600 })), { brand: 'qt ', durationSec: 61 });+});++test('mp4Info finds a moov that sits after the media data, even behind a 64-bit mdat', () => {+ assert.equal(info(M.mp4({ moovAtEnd: true, seconds: 30, mdatBytes: 200000 })).durationSec, 30);+ assert.equal(info(M.mp4({ moovAtEnd: true, seconds: 7, bigMdat: true })).durationSec, 7);+});++test('a fragmented file with no length in mvhd is read from mehd, and one with neither is unknown', () => {+ const frag = M.mp4({ seconds: 0, extraMoov: [M.box('mvex', M.mehd(95000))] });+ assert.equal(info(frag).durationSec, 95);+ assert.equal(info(M.mp4({ seconds: 0 })).durationSec, null);+});++test('mp4Info refuses what is not a well-formed file', () => {+ const noMoov = Buffer.concat([M.ftyp(), M.box('mdat', Buffer.alloc(100))]);+ assert.equal(info(noMoov), null, 'no moov');+ const noMvhd = Buffer.concat([M.ftyp(), M.box('moov', M.box('trak', Buffer.alloc(20)))]);+ assert.equal(info(noMvhd), null, 'no mvhd');+ assert.equal(info(M.mp4({ timescale: 0 })), null, 'zero timescale');+ assert.equal(info(Buffer.concat([M.box('free', Buffer.alloc(8)), M.mp4()])), null, 'ftyp not first');+ // A moov over 8 MB: no real clip has one, and walking it is what an attacker would want.+ assert.equal(info(M.mp4({ moovPad: 8 * 1024 * 1024 + 10 })), null, 'moov > 8 MB');+ // A box that claims more bytes than its parent has.+ const lying = M.mp4();+ lying.writeUInt32BE(lying.length * 2, M.ftyp().length);+ assert.equal(info(lying), null, 'box overruns the file');+ // A size smaller than its own header.+ const tiny = M.mp4();+ tiny.writeUInt32BE(4, M.ftyp().length);+ assert.equal(info(tiny), null, 'box smaller than its header');+});++test('mp4Info gives up after 10k boxes instead of walking forever', () => {+ const many = Buffer.concat([M.ftyp(), ...Array.from({ length: 10050 }, () => M.box('free')), M.box('moov', M.mvhd())]);+ assert.equal(info(many), null);+});