Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix: keep queued Loom imports out of the shared job store
Richie McIlroy94aaeb0b1c15
diff --git a/apps/media-server/src/__tests__/loom-importer/app.test.ts b/apps/media-server/src/__tests__/loom-importer/app.test.tsnew file mode 100644index 000000000..93dec5a6f--- /dev/null+++ b/apps/media-server/src/__tests__/loom-importer/app.test.ts@@ -0,0 +1,114 @@+import { afterAll, beforeAll, describe, expect, test } from "bun:test";+import { mkdtemp, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { getJob } from "../../lib/job-manager";+import {+ createLoomImporter,+ loadImporterSettings,+} from "../../loom-importer/app";++describe("Loom importer API", () => {+ let root: string;+ let slowSource: ReturnType<typeof Bun.serve>;+ let release: () => void = () => {};+ const released = new Promise<void>((resolve) => {+ release = resolve;+ });++ beforeAll(async () => {+ process.env.MEDIA_SERVER_WEBHOOK_SECRET = "importer-secret";+ root = await mkdtemp(join(tmpdir(), "loom-importer-app-"));+ slowSource = Bun.serve({+ port: 0,+ hostname: "127.0.0.1",+ async fetch() {+ await released;+ return new Response("#EXTM3U\n#EXT-X-ENDLIST");+ },+ });+ });++ afterAll(async () => {+ release();+ slowSource?.stop(true);+ if (root) await rm(root, { recursive: true, force: true });+ });++ function importer() {+ const config = loadImporterSettings({+ LOOM_IMPORTER_WORK_DIR: root,+ LOOM_IMPORTER_ALLOWED_HOSTS: "127.0.0.1",+ LOOM_IMPORTER_MAX_JOBS: "1",+ LOOM_IMPORTER_MAX_QUEUED: "1",+ LOOM_IMPORTER_MIN_FREE_GB: "1",+ });+ config.settings.minFreeBytes = 1;+ return createLoomImporter(config);+ }++ const body = (videoId: string) =>+ JSON.stringify({+ videoId,+ userId: "user",+ videoUrl: `http://127.0.0.1:${slowSource.port}/master.m3u8`,+ outputPresignedUrl: "http://127.0.0.1:9/result.mp4",+ });++ const post = (+ app: ReturnType<typeof importer>["app"],+ payload: string,+ secret = "importer-secret",+ ) =>+ app.request("/loom-importer/video/import", {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ "x-media-server-secret": secret,+ },+ body: payload,+ });++ test("requires the media server secret", async () => {+ const { app } = importer();+ expect((await post(app, body("a"), "wrong")).status).toBe(401);+ expect((await app.request("/loom-importer/video/status")).status).toBe(401);+ });++ test("keeps waiting imports out of the shared job store and reports them as queued", async () => {+ const { app, stats } = importer();+ const first = (await (await post(app, body("first"))).json()) as {+ jobId: string;+ };+ const second = (await (await post(app, body("second"))).json()) as {+ jobId: string;+ };+ expect(stats()).toEqual({ running: 1, queued: 1 });+ expect(getJob(first.jobId)?.abortController).toBeDefined();+ expect(getJob(second.jobId)).toBeUndefined();+ const status = await app.request(+ `/loom-importer/video/process/${second.jobId}/status`,+ { headers: { "x-media-server-secret": "importer-secret" } },+ );+ expect(await status.json()).toMatchObject({+ jobId: second.jobId,+ videoId: "second",+ phase: "queued",+ });+ const full = await post(app, body("third"));+ expect(full.status).toBe(503);+ expect(await full.json()).toMatchObject({ code: "SERVER_BUSY" });+ release();+ const deadline = Date.now() + 10_000;+ while (stats().running + stats().queued > 0 && Date.now() < deadline)+ await Bun.sleep(50);+ expect(getJob(first.jobId)?.phase).toBe("error");+ expect(getJob(second.jobId)?.phase).toBe("error");+ });++ test("rejects malformed jobs", async () => {+ const { app } = importer();+ const response = await post(app, JSON.stringify({ videoId: "x" }));+ expect(response.status).toBe(400);+ });+});diff --git a/apps/media-server/src/loom-importer/app.ts b/apps/media-server/src/loom-importer/app.tsindex 709add741..79cac4a38 100644--- a/apps/media-server/src/loom-importer/app.ts+++ b/apps/media-server/src/loom-importer/app.ts@@ -4,13 +4,14 @@ import { Hono } from "hono";import { validateMediaServerSecret } from "../lib/auth";import {createJob,- deleteJob,+ extendJobLifetime,generateJobId,getAllJobs,getJob,getJobProgress,type Job,sendWebhook,+ touchJob,updateJob,} from "../lib/job-manager";import { LOOM_IMPORT_MAX_BYTES } from "../lib/media-video";@@ -55,7 +56,13 @@ export function loadImporterSettings(};}-const TERMINAL_JOB_TTL_MS = 24 * 60 * 60 * 1000;+const JOB_LIFETIME_MS = 4 * 60 * 60 * 1000;+const JOB_HEARTBEAT_MS = 60_000;++interface QueuedImport {+ jobId: string;+ request: ImportRequest;+}function isActive(job: Job) {return (@@ -68,28 +75,41 @@ function isActive(job: Job) {export function createLoomImporter(config: ReturnType<typeof loadImporterSettings> = loadImporterSettings(),) {- const queue: Array<{ job: Job; request: ImportRequest }> = [];+ const queue: QueuedImport[] = [];const running = new Map<string, AbortController>();let draining = false;+ const startJob = ({ jobId, request }: QueuedImport) =>+ createJob(+ jobId,+ request.videoId,+ request.userId,+ request.webhookUrl,+ request.webhookSecret,+ );+const pump = () => {while (running.size < config.maxActiveJobs && queue.length > 0) {const next = queue.shift();if (!next) break;+ const job = startJob(next);const controller = new AbortController();- running.set(next.job.jobId, controller);+ updateJob(job.jobId, { abortController: controller });+ extendJobLifetime(job.jobId, JOB_LIFETIME_MS);+ const heartbeat = setInterval(+ () => touchJob(job.jobId),+ JOB_HEARTBEAT_MS,+ );+ running.set(job.jobId, controller);void runLoomImport(- next.job,+ job,next.request,config.settings,new Semaphore(config.settings.jobConnections),controller.signal,).finally(() => {- running.delete(next.job.jobId);- setTimeout(- () => deleteJob(next.job.jobId),- TERMINAL_JOB_TTL_MS,- ).unref?.();+ clearInterval(heartbeat);+ running.delete(job.jobId);pump();});}@@ -158,18 +178,11 @@ export function createLoomImporter(503,);}- const request = parsed.data;- const job = createJob(- generateJobId(),- request.videoId,- request.userId,- request.webhookUrl,- request.webhookSecret,- );- queue.push({ job, request });+ const jobId = generateJobId();+ queue.push({ jobId, request: parsed.data });pump();return c.json({- jobId: job.jobId,+ jobId,status: "queued",message: "Loom import queued",});@@ -178,9 +191,19 @@ export function createLoomImporter(app.get("/video/process/:jobId/status", (c) => {if (!validateMediaServerSecret(c))return c.json({ error: "Unauthorized" }, 401);- const job = getJob(c.req.param("jobId"));- if (!job) return c.json({ error: "Job not found", code: "NOT_FOUND" }, 404);- return c.json(getJobProgress(job));+ const jobId = c.req.param("jobId");+ const job = getJob(jobId);+ if (job) return c.json(getJobProgress(job));+ const queued = queue.find((entry) => entry.jobId === jobId);+ if (queued)+ return c.json({+ jobId,+ videoId: queued.request.videoId,+ phase: "queued",+ progress: 0,+ message: "Waiting for a Loom import slot",+ });+ return c.json({ error: "Job not found", code: "NOT_FOUND" }, 404);});const cancelJob = async (job: Job) => {@@ -197,7 +220,9 @@ export function createLoomImporter(const deadline = Date.now() + timeoutMs;while ((queue.length > 0 || running.size > 0) && Date.now() < deadline)await new Promise((resolve) => setTimeout(resolve, 500));- await Promise.all(queue.splice(0).map(({ job }) => cancelJob(job)));+ await Promise.all(+ queue.splice(0).map((entry) => cancelJob(startJob(entry))),+ );for (const controller of running.values()) controller.abort();while (running.size > 0)await new Promise((resolve) => setTimeout(resolve, 250));