Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
feat(media-server): optional parallel ranged source downloads
Richie McIlroy7da9668f7da4
diff --git a/apps/media-server/Dockerfile b/apps/media-server/Dockerfileindex c09234479..05fc8842d 100644--- a/apps/media-server/Dockerfile+++ b/apps/media-server/Dockerfile@@ -15,7 +15,7 @@ COPY apps/media-server/capcodec.md ./capcodec.mdRUN bun test src/__tests__/lib/audio-quality-policy.test.ts src/__tests__/lib/audio-quality.integration.test.ts \&& bun test src/__tests__/lib/audio-quality-formats.integration.test.ts src/__tests__/lib/audio-quality-benchmark.test.ts \&& bun test src/__tests__/lib/audio-levels.integration.test.ts \- && bun test src/__tests__/lib/drive-resumable-upload.test.ts src/__tests__/lib/storage-upload.test.ts src/__tests__/lib/container-memory.test.ts \+ && bun test src/__tests__/lib/drive-resumable-upload.test.ts src/__tests__/lib/storage-upload.test.ts src/__tests__/lib/container-memory.test.ts src/__tests__/lib/parallel-download.test.ts \&& bun test src/__tests__/lib/media-size.test.ts src/__tests__/lib/media-probe.integration.test.ts \&& bun test src/__tests__/lib/media-audio.integration.test.ts \&& bun test src/__tests__/lib/recording-verification.integration.test.ts src/__tests__/lib/job-manager.test.ts \diff --git a/apps/media-server/src/__tests__/lib/parallel-download.test.ts b/apps/media-server/src/__tests__/lib/parallel-download.test.tsnew file mode 100644index 000000000..a956882ee--- /dev/null+++ b/apps/media-server/src/__tests__/lib/parallel-download.test.ts@@ -0,0 +1,137 @@+import { afterAll, beforeAll, describe, expect, test } from "bun:test";+import { randomBytes } from "node:crypto";+import { mkdtemp, readFile, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import {+ downloadUrlToPathParallel,+ parallelDownloadOptionsFromEnv,+ planRanges,+} from "../../lib/parallel-download";++const MB = 1024 * 1024;+const payload = randomBytes(5 * MB + 12345);+let dir: string;+let server: ReturnType<typeof Bun.serve>;+let rangeRequests = 0;++beforeAll(async () => {+ dir = await mkdtemp(join(tmpdir(), "parallel-download-"));+ server = Bun.serve({+ port: 0,+ fetch(req) {+ const url = new URL(req.url);+ const range = req.headers.get("range")?.match(/^bytes=(\d+)-(\d+)$/);+ if (url.pathname === "/plain" || !range) {+ return new Response(payload, { headers: { ETag: '"v1"' } });+ }+ rangeRequests++;+ if (url.pathname === "/changed" && Number(range[1]) > 0) {+ return new Response("precondition", { status: 412 });+ }+ const start = Number(range[1]);+ const end = Math.min(Number(range[2]), payload.length - 1);+ const body =+ url.pathname === "/short" && start > 0+ ? payload.subarray(start, end)+ : payload.subarray(start, end + 1);+ return new Response(body, {+ status: 206,+ headers: {+ ETag: '"v1"',+ "Content-Range": `bytes ${start}-${end}/${payload.length}`,+ },+ });+ },+ });+});++afterAll(async () => {+ server.stop(true);+ await rm(dir, { recursive: true, force: true });+});++describe("parallel download", () => {+ test("env defaults keep the single-stream path", () => {+ expect(parallelDownloadOptionsFromEnv({})).toEqual({+ concurrency: 1,+ partBytes: 16 * MB,+ });+ expect(+ parallelDownloadOptionsFromEnv({+ MEDIA_SERVER_DOWNLOAD_CONCURRENCY: "8",+ MEDIA_SERVER_DOWNLOAD_PART_MB: "32",+ }),+ ).toEqual({ concurrency: 8, partBytes: 32 * MB });+ expect(+ parallelDownloadOptionsFromEnv({+ MEDIA_SERVER_DOWNLOAD_CONCURRENCY: "500",+ }).concurrency,+ ).toBe(32);+ });++ test("plans contiguous ranges after the first part", () => {+ expect(planRanges(10, 4, 3)).toEqual([+ { start: 4, end: 6 },+ { start: 7, end: 9 },+ ]);+ expect(planRanges(4, 4, 3)).toEqual([]);+ });++ test("reassembles ranged parts byte for byte", async () => {+ const path = join(dir, "ranged.bin");+ const result = await downloadUrlToPathParallel(+ `http://localhost:${server.port}/file`,+ path,+ { concurrency: 4, partBytes: MB },+ );+ expect(result.ranged).toBe(true);+ expect(result.parts).toBe(6);+ expect(Buffer.compare(await readFile(path), payload)).toBe(0);+ });++ test("falls back to one stream when ranges are unsupported", async () => {+ const path = join(dir, "plain.bin");+ const result = await downloadUrlToPathParallel(+ `http://localhost:${server.port}/plain`,+ path,+ { concurrency: 4, partBytes: MB },+ );+ expect(result).toEqual({ bytes: payload.length, parts: 1, ranged: false });+ expect(Buffer.compare(await readFile(path), payload)).toBe(0);+ });++ test("fails when the object changes mid-download", async () => {+ await expect(+ downloadUrlToPathParallel(+ `http://localhost:${server.port}/changed`,+ join(dir, "changed.bin"),+ { concurrency: 4, partBytes: MB },+ ),+ ).rejects.toThrow("Ranged download part failed: 412");+ });++ test("fails when a part comes back short", async () => {+ await expect(+ downloadUrlToPathParallel(+ `http://localhost:${server.port}/short`,+ join(dir, "short.bin"),+ { concurrency: 4, partBytes: MB },+ ),+ ).rejects.toThrow("expected");+ });++ test("stops on abort", async () => {+ const controller = new AbortController();+ controller.abort();+ const before = rangeRequests;+ await expect(+ downloadUrlToPathParallel(+ `http://localhost:${server.port}/file`,+ join(dir, "aborted.bin"),+ { concurrency: 4, partBytes: MB, signal: controller.signal },+ ),+ ).rejects.toThrow();+ expect(rangeRequests).toBe(before);+ });+});diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.tsindex 481589793..d2645568e 100644--- a/apps/media-server/src/lib/media-video.ts+++ b/apps/media-server/src/lib/media-video.ts@@ -21,6 +21,10 @@ import {} from "./media-common";import { probeVideoFile } from "./media-probe";import { fetchMedia, materializeMedia } from "./media-transfer";+import {+ downloadUrlToPathParallel,+ parallelDownloadOptionsFromEnv,+} from "./parallel-download";import { readRecordingAudioTail } from "./recording-packet-proof";import {RecordingTimingError,@@ -1252,6 +1256,18 @@ export async function downloadVideoToTemp(? AbortSignal.any([abortSignal, timeoutSignal]): timeoutSignal;+ const parallel = parallelDownloadOptionsFromEnv();+ if (parallel.concurrency > 1) {+ await downloadUrlToPathParallel(videoUrl, tempFile.path, {+ ...parallel,+ signal: combinedSignal,+ });+ if (file(tempFile.path).size === 0) {+ throw new Error("Downloaded video is empty");+ }+ return tempFile;+ }+const response = await fetchMedia(videoUrl, {signal: combinedSignal,});diff --git a/apps/media-server/src/lib/parallel-download.ts b/apps/media-server/src/lib/parallel-download.tsnew file mode 100644index 000000000..443bb31d6--- /dev/null+++ b/apps/media-server/src/lib/parallel-download.ts@@ -0,0 +1,173 @@+import { open } from "node:fs/promises";++const MIN_PART_BYTES = 1024 * 1024;++export interface ParallelDownloadOptions {+ concurrency: number;+ partBytes: number;+ signal?: AbortSignal;+ fetchImpl?: typeof fetch;+}++export interface ParallelDownloadResult {+ bytes: number;+ parts: number;+ ranged: boolean;+}++export function parallelDownloadOptionsFromEnv(+ env: Record<string, string | undefined> = process.env,+): Pick<ParallelDownloadOptions, "concurrency" | "partBytes"> {+ const concurrency = Number.parseInt(+ env.MEDIA_SERVER_DOWNLOAD_CONCURRENCY ?? "",+ 10,+ );+ const partMb = Number.parseInt(env.MEDIA_SERVER_DOWNLOAD_PART_MB ?? "", 10);+ return {+ concurrency:+ Number.isFinite(concurrency) && concurrency > 0+ ? Math.min(concurrency, 32)+ : 1,+ partBytes: Math.max(+ MIN_PART_BYTES,+ (Number.isFinite(partMb) && partMb > 0 ? partMb : 16) * 1024 * 1024,+ ),+ };+}++export function planRanges(+ totalBytes: number,+ firstPartBytes: number,+ partBytes: number,+): Array<{ start: number; end: number }> {+ const ranges: Array<{ start: number; end: number }> = [];+ for (let start = firstPartBytes; start < totalBytes; start += partBytes) {+ ranges.push({ start, end: Math.min(totalBytes, start + partBytes) - 1 });+ }+ return ranges;+}++function parseTotalFromContentRange(value: string | null): number | null {+ const match = value?.match(/^bytes\s+(\d+)-(\d+)\/(\d+)$/i);+ if (!match) return null;+ const total = Number(match[3]);+ return Number.isSafeInteger(total) ? total : null;+}++async function readInto(+ response: Response,+ write: (chunk: Uint8Array, offset: number) => Promise<void>,+ start: number,+ expected: number | null,+): Promise<number> {+ if (!response.body) throw new Error("No response body");+ const reader = response.body.getReader();+ let offset = start;+ try {+ while (true) {+ const { done, value } = await reader.read();+ if (done) break;+ await write(value, offset);+ offset += value.byteLength;+ }+ } finally {+ reader.releaseLock();+ }+ const read = offset - start;+ if (expected !== null && read !== expected) {+ throw new Error(+ `Ranged download returned ${read} bytes, expected ${expected}`,+ );+ }+ return read;+}++export async function downloadUrlToPathParallel(+ url: string,+ path: string,+ options: ParallelDownloadOptions,+): Promise<ParallelDownloadResult> {+ const fetchImpl = options.fetchImpl ?? fetch;+ const partBytes = Math.max(MIN_PART_BYTES, options.partBytes);+ const handle = await open(path, "w");+ const write = async (chunk: Uint8Array, offset: number) => {+ let written = 0;+ while (written < chunk.byteLength) {+ const { bytesWritten } = await handle.write(+ chunk,+ written,+ chunk.byteLength - written,+ offset + written,+ );+ written += bytesWritten;+ }+ };+ try {+ const first = await fetchImpl(url, {+ headers: { Range: `bytes=0-${partBytes - 1}` },+ signal: options.signal,+ });+ if (!first.ok) {+ const body = await first.text().catch(() => "");+ throw new Error(+ `Failed to download video: ${first.status} ${first.statusText}; ${body.slice(0, 300)}`,+ );+ }+ const total =+ first.status === 206+ ? parseTotalFromContentRange(first.headers.get("content-range"))+ : null;+ if (total === null) {+ const bytes = await readInto(first, write, 0, null);+ return { bytes, parts: 1, ranged: false };+ }+ const firstBytes = Math.min(partBytes, total);+ const etag = first.headers.get("etag");+ const ranges = planRanges(total, firstBytes, partBytes);+ const controller = new AbortController();+ const signal = options.signal+ ? AbortSignal.any([options.signal, controller.signal])+ : controller.signal;+ let next = 0;+ const worker = async () => {+ while (next < ranges.length) {+ const range = ranges[next++];+ if (!range) return;+ const response = await fetchImpl(url, {+ headers: {+ Range: `bytes=${range.start}-${range.end}`,+ ...(etag ? { "If-Match": etag } : {}),+ },+ signal,+ });+ if (response.status !== 206) {+ await response.body?.cancel().catch(() => {});+ throw new Error(+ `Ranged download part failed: ${response.status} ${response.statusText}`,+ );+ }+ await readInto(+ response,+ write,+ range.start,+ range.end - range.start + 1,+ );+ }+ };+ const workers = Array.from(+ { length: Math.max(1, Math.min(options.concurrency - 1, ranges.length)) },+ () => worker(),+ );+ const firstRead = readInto(first, write, 0, firstBytes);+ try {+ await Promise.all([firstRead, ...workers]);+ } catch (error) {+ controller.abort();+ await Promise.allSettled([firstRead, ...workers]);+ throw error;+ }+ return { bytes: total, parts: ranges.length + 1, ranged: true };+ } finally {+ await handle.close();+ }+}