Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
feat(media-server): give customer Loom imports their own importer lane
Richie McIlroy8e19f1aabca9
diff --git a/apps/media-server/src/__tests__/loom-importer/priority.test.ts b/apps/media-server/src/__tests__/loom-importer/priority.test.tsnew file mode 100644index 000000000..c662cfdec--- /dev/null+++ b/apps/media-server/src/__tests__/loom-importer/priority.test.ts@@ -0,0 +1,244 @@+import { afterAll, beforeAll, describe, expect, test } from "bun:test";+import { mkdtemp, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { getJob } from "../../lib/job-manager";+import {+ createLoomImporter,+ loadImporterSettings,+} from "../../loom-importer/app";+import { Semaphore } from "../../loom-importer/fetch";++describe("Semaphore priority", () => {+ test("hands a freed permit to priority waiters before earlier normal waiters", async () => {+ const semaphore = new Semaphore(1);+ const order: string[] = [];+ let releaseHolder: () => void = () => {};+ const holder = semaphore.run(+ () =>+ new Promise<void>((resolve) => {+ releaseHolder = resolve;+ }),+ );+ const waiters = [+ semaphore.run(async () => {+ order.push("bulk-1");+ }),+ semaphore.run(async () => {+ order.push("bulk-2");+ }),+ semaphore.run(async () => {+ order.push("customer");+ }, true),+ ];+ releaseHolder();+ await Promise.all([holder, ...waiters]);+ expect(order).toEqual(["customer", "bulk-1", "bulk-2"]);+ });++ test("keeps first-in first-out order when nobody asks for priority", async () => {+ const semaphore = new Semaphore(1);+ const order: number[] = [];+ await Promise.all(+ Array.from({ length: 5 }, (_, index) =>+ semaphore.run(async () => {+ order.push(index);+ await Bun.sleep(1);+ }),+ ),+ );+ expect(order).toEqual([0, 1, 2, 3, 4]);+ });+});++describe("Loom importer lanes", () => {+ let root: string;+ let source: ReturnType<typeof Bun.serve>;+ const releases = new Map<string, () => void>();+ const gates = new Map<string, Promise<void>>();+ const started = new Set<string>();+ const jobIds = new Map<string, string>();++ beforeAll(async () => {+ process.env.MEDIA_SERVER_WEBHOOK_SECRET = "importer-secret";+ root = await mkdtemp(join(tmpdir(), "loom-importer-lanes-"));+ source = Bun.serve({+ port: 0,+ hostname: "127.0.0.1",+ async fetch(request) {+ const videoId = new URL(request.url).pathname.split("/")[1] ?? "";+ started.add(videoId);+ await gates.get(videoId);+ return new Response("#EXTM3U\n#EXT-X-ENDLIST");+ },+ });+ });++ afterAll(async () => {+ for (const release of releases.values()) release();+ source?.stop(true);+ if (root) await rm(root, { recursive: true, force: true });+ });++ type Importer = ReturnType<typeof createLoomImporter>;++ function importer(env: Record<string, string>) {+ const config = loadImporterSettings({+ LOOM_IMPORTER_WORK_DIR: root,+ LOOM_IMPORTER_ALLOWED_HOSTS: "127.0.0.1",+ LOOM_IMPORTER_MIN_FREE_GB: "1",+ ...env,+ });+ config.settings.minFreeBytes = 1;+ return createLoomImporter(config);+ }++ async function post(+ { app }: Importer,+ videoId: string,+ importPriority?: "customer" | "bulk",+ ) {+ gates.set(+ videoId,+ new Promise<void>((resolve) => releases.set(videoId, resolve)),+ );+ const response = await app.request("/loom-importer/video/import", {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ "x-media-server-secret": "importer-secret",+ },+ body: JSON.stringify({+ videoId,+ userId: "user",+ videoUrl: `http://127.0.0.1:${source.port}/${videoId}/master.m3u8`,+ outputPresignedUrl: "http://127.0.0.1:9/result.mp4",+ ...(importPriority ? { importPriority } : {}),+ }),+ });+ if (response.ok)+ jobIds.set(videoId, ((await response.json()) as { jobId: string }).jobId);+ return response;+ }++ const isFinished = (videoId: string) => {+ const jobId = jobIds.get(videoId);+ const phase = jobId ? getJob(jobId)?.phase : undefined;+ return phase === "error" || phase === "complete" || phase === "cancelled";+ };++ async function waitFor(condition: () => boolean) {+ const deadline = Date.now() + 10_000;+ while (!condition() && Date.now() < deadline) await Bun.sleep(10);+ expect(condition()).toBe(true);+ }++ async function expectRunning(+ importer: Importer,+ ids: string[],+ runningCustomer: number,+ ) {+ await waitFor(() => ids.every((id) => started.has(id)));+ await Bun.sleep(100);+ const status = (await (+ await importer.app.request("/loom-importer/video/status", {+ headers: { "x-media-server-secret": "importer-secret" },+ })+ ).json()) as { runningCustomer: number };+ expect(importer.stats().running).toBe(ids.length);+ expect(status.runningCustomer).toBe(runningCustomer);+ }++ function expectWaiting(ids: string[]) {+ for (const id of ids) expect(started.has(id)).toBe(false);+ }++ async function finish(videoId: string) {+ releases.get(videoId)?.();+ await waitFor(() => isFinished(videoId));+ }++ async function settle(importer: Importer) {+ for (const release of releases.values()) release();+ await waitFor(+ () => importer.stats().running + importer.stats().queued === 0,+ );+ }++ test("bulk imports keep every general slot and never take a customer slot", async () => {+ const box = importer({+ LOOM_IMPORTER_MAX_JOBS: "2",+ LOOM_IMPORTER_CUSTOMER_JOBS: "3",+ });+ for (const id of ["b1", "b2", "b3", "b4"]) await post(box, id, "bulk");+ await expectRunning(box, ["b1", "b2"], 0);+ expectWaiting(["b3", "b4"]);+ expect(box.stats().queued).toBe(2);+ await finish("b1");+ await expectRunning(box, ["b2", "b3"], 0);+ expectWaiting(["b4"]);+ await settle(box);+ });++ test("imports without a lane run as bulk", async () => {+ const box = importer({+ LOOM_IMPORTER_MAX_JOBS: "1",+ LOOM_IMPORTER_CUSTOMER_JOBS: "1",+ });+ await post(box, "u1");+ await post(box, "u2");+ await expectRunning(box, ["u1"], 0);+ expectWaiting(["u2"]);+ await settle(box);+ });++ test("a customer import starts at once while bulk work fills the box", async () => {+ const box = importer({+ LOOM_IMPORTER_MAX_JOBS: "2",+ LOOM_IMPORTER_CUSTOMER_JOBS: "1",+ });+ for (const id of ["f1", "f2", "f3"]) await post(box, id, "bulk");+ await expectRunning(box, ["f1", "f2"], 0);+ await post(box, "c1", "customer");+ await expectRunning(box, ["f1", "f2", "c1"], 1);+ expectWaiting(["f3"]);+ await finish("c1");+ await expectRunning(box, ["f1", "f2"], 0);+ expectWaiting(["f3"]);+ await settle(box);+ });++ test("queued customer imports go before queued bulk imports", async () => {+ const box = importer({+ LOOM_IMPORTER_MAX_JOBS: "1",+ LOOM_IMPORTER_CUSTOMER_JOBS: "1",+ });+ await post(box, "g1", "bulk");+ await post(box, "g2", "bulk");+ await post(box, "k1", "customer");+ await post(box, "k2", "customer");+ await expectRunning(box, ["g1", "k1"], 1);+ expectWaiting(["g2", "k2"]);+ await finish("g1");+ await expectRunning(box, ["k1", "k2"], 2);+ expectWaiting(["g2"]);+ await finish("k1");+ await expectRunning(box, ["k2", "g2"], 1);+ await settle(box);+ });++ test("a full bulk queue does not turn customers away", async () => {+ const box = importer({+ LOOM_IMPORTER_MAX_JOBS: "1",+ LOOM_IMPORTER_CUSTOMER_JOBS: "1",+ LOOM_IMPORTER_MAX_QUEUED: "1",+ });+ expect((await post(box, "q1", "bulk")).status).toBe(200);+ expect((await post(box, "q2", "bulk")).status).toBe(200);+ expect((await post(box, "q3", "bulk")).status).toBe(503);+ expect((await post(box, "q4", "customer")).status).toBe(200);+ expect((await post(box, "q5", "customer")).status).toBe(200);+ expect((await post(box, "q6", "customer")).status).toBe(503);+ await settle(box);+ });+});diff --git a/apps/media-server/src/loom-importer/app.ts b/apps/media-server/src/loom-importer/app.tsindex 2a85c79e8..3b454fb90 100644--- a/apps/media-server/src/loom-importer/app.ts+++ b/apps/media-server/src/loom-importer/app.ts@@ -53,6 +53,7 @@ export function loadImporterSettings(return {settings,maxActiveJobs: positiveInt(env.LOOM_IMPORTER_MAX_JOBS, 16),+ maxCustomerJobs: positiveInt(env.LOOM_IMPORTER_CUSTOMER_JOBS, 16),maxQueuedJobs: positiveInt(env.LOOM_IMPORTER_MAX_QUEUED, 400),maxPreviewJobs: positiveInt(env.LOOM_IMPORTER_PREVIEW_JOBS, 4),};@@ -67,6 +68,11 @@ interface QueuedImport {request: ImportRequest;}+type ImportLane = "customer" | "bulk";++const laneOf = (request: ImportRequest): ImportLane =>+ request.importPriority === "customer" ? "customer" : "bulk";+function isActive(job: Job) {return (job.phase !== "complete" &&@@ -78,8 +84,14 @@ function isActive(job: Job) {export function createLoomImporter(config: ReturnType<typeof loadImporterSettings> = loadImporterSettings(),) {- const queue: QueuedImport[] = [];+ const queues: Record<ImportLane, QueuedImport[]> = {+ customer: [],+ bulk: [],+ };+ const queuedCount = () => queues.customer.length + queues.bulk.length;+ const queuedEntries = () => [...queues.customer, ...queues.bulk];const running = new Map<string, AbortController>();+ let runningCustomer = 0;let draining = false;const previewSemaphore = new Semaphore(config.maxPreviewJobs);const previews = new Set<AbortController>();@@ -93,10 +105,24 @@ export function createLoomImporter(request.webhookSecret,);+ const canStart = (lane: ImportLane) =>+ running.size <+ config.maxActiveJobs ++ (lane === "customer"+ ? config.maxCustomerJobs+ : Math.min(runningCustomer, config.maxCustomerJobs));++ const nextImport = () => {+ if (queues.customer.length > 0 && canStart("customer"))+ return queues.customer.shift();+ if (queues.bulk.length > 0 && canStart("bulk")) return queues.bulk.shift();+ return undefined;+ };+const pump = () => {- while (running.size < config.maxActiveJobs && queue.length > 0) {- const next = queue.shift();- if (!next) break;+ for (let next = nextImport(); next; next = nextImport()) {+ const customer = laneOf(next.request) === "customer";+ if (customer) runningCustomer++;const job = startJob(next);const controller = new AbortController();updateJob(job.jobId, { abortController: controller });@@ -114,6 +140,7 @@ export function createLoomImporter(controller.signal,).finally(() => {clearInterval(heartbeat);+ if (customer) runningCustomer--;running.delete(job.jobId);pump();});@@ -126,7 +153,7 @@ export function createLoomImporter(c.json({status: draining ? "draining" : "ok",running: running.size,- queued: queue.length,+ queued: queuedCount(),}),);@@ -140,8 +167,11 @@ export function createLoomImporter(loomDeferredOriginals: true,draining,running: running.size,- queued: queue.length,+ queued: queuedCount(),+ runningCustomer,+ queuedCustomer: queues.customer.length,maxActiveJobs: config.maxActiveJobs,+ maxCustomerJobs: config.maxCustomerJobs,maxQueuedJobs: config.maxQueuedJobs,jobs: getAllJobs().filter(isActive)@@ -153,7 +183,7 @@ export function createLoomImporter(ageSeconds: Math.round((Date.now() - job.createdAt) / 1000),})).concat(- queue.map((entry) => ({+ queuedEntries().map((entry) => ({jobId: entry.jobId,videoId: entry.request.videoId,phase: "queued" as Job["phase"],@@ -179,7 +209,8 @@ export function createLoomImporter(},400,);- if (draining || queue.length >= config.maxQueuedJobs) {+ const lane = laneOf(parsed.data);+ if (draining || queues[lane].length >= config.maxQueuedJobs) {c.header("Retry-After", "30");return c.json({@@ -193,7 +224,7 @@ export function createLoomImporter();}const jobId = generateJobId();- queue.push({ jobId, request: parsed.data });+ queues[lane].push({ jobId, request: parsed.data });pump();return c.json({jobId,@@ -246,7 +277,7 @@ export function createLoomImporter(const jobId = c.req.param("jobId");const job = getJob(jobId);if (job) return c.json(getJobProgress(job));- const queued = queue.find((entry) => entry.jobId === jobId);+ const queued = queuedEntries().find((entry) => entry.jobId === jobId);if (queued)return c.json({jobId,@@ -271,13 +302,15 @@ export function createLoomImporter(draining = true;const deadline = Date.now() + timeoutMs;while (- (queue.length > 0 || running.size > 0 || previews.size > 0) &&+ (queuedCount() > 0 || running.size > 0 || previews.size > 0) &&Date.now() < deadline)await new Promise((resolve) => setTimeout(resolve, 500));for (const controller of previews) controller.abort();await Promise.all(- queue.splice(0).map((entry) => cancelJob(startJob(entry))),+ [...queues.customer.splice(0), ...queues.bulk.splice(0)].map((entry) =>+ cancelJob(startJob(entry)),+ ),);for (const controller of running.values()) controller.abort();while (running.size > 0 || previews.size > 0)@@ -287,6 +320,6 @@ export function createLoomImporter(return {app,drain,- stats: () => ({ running: running.size, queued: queue.length }),+ stats: () => ({ running: running.size, queued: queuedCount() }),};}diff --git a/apps/media-server/src/loom-importer/fetch.ts b/apps/media-server/src/loom-importer/fetch.tsindex e01431feb..bdb4f2b40 100644--- a/apps/media-server/src/loom-importer/fetch.ts+++ b/apps/media-server/src/loom-importer/fetch.ts@@ -5,21 +5,24 @@ export class LoomSourceRejectedError extends Error {}export class Semaphore {private available: number;private readonly waiters: Array<() => void> = [];+ private readonly priorityWaiters: Array<() => void> = [];constructor(size: number) {this.available = Math.max(1, size);}- async run<T>(work: () => Promise<T>): Promise<T> {+ async run<T>(work: () => Promise<T>, priority = false): Promise<T> {if (this.available > 0) {this.available--;} else {- await new Promise<void>((resolve) => this.waiters.push(resolve));+ await new Promise<void>((resolve) =>+ (priority ? this.priorityWaiters : this.waiters).push(resolve),+ );}try {return await work();} finally {- const next = this.waiters.shift();+ const next = this.priorityWaiters.shift() ?? this.waiters.shift();if (next) next();else this.available++;}diff --git a/apps/media-server/src/loom-importer/job.ts b/apps/media-server/src/loom-importer/job.tsindex 2d34c8ae6..b21c52006 100644--- a/apps/media-server/src/loom-importer/job.ts+++ b/apps/media-server/src/loom-importer/job.ts@@ -89,6 +89,7 @@ export const importRequestSchema = z.object({previewGifPresignedUrl: z.string().url().optional(),webhookUrl: z.string().url().optional(),webhookSecret: z.string().optional(),+ importPriority: z.enum(["customer", "bulk"]).optional(),});export type ImportRequest = z.infer<typeof importRequestSchema>;@@ -221,6 +222,7 @@ async function prepareSource(requestTimeoutMs: settings.segmentTimeoutMs,jobSemaphore,globalSemaphore: settings.globalSemaphore,+ priority: request.importPriority === "customer",maxBytes,onProgress: (progress) => {if (diff --git a/apps/media-server/src/loom-importer/streaming.ts b/apps/media-server/src/loom-importer/streaming.tsindex 06ddd19c8..d6935c0e0 100644--- a/apps/media-server/src/loom-importer/streaming.ts+++ b/apps/media-server/src/loom-importer/streaming.ts@@ -176,6 +176,7 @@ export interface PrefetchOptions extends Omit<DownloadToFileOptions, "signal"> {signal: AbortSignal;jobSemaphore: Semaphore;globalSemaphore: Semaphore;+ priority?: boolean;maxBytes: number;onProgress?: (progress: PrefetchProgress) => void;}@@ -247,7 +248,7 @@ export async function prefetchMaterializedHls(throw error;}options.onProgress?.({ ...progress });- }),+ }, options.priority),).catch((error) => {failure.abort(error);