Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

feat: add a dedicated Loom importer entrypoint to the media server

Richie McIlroy7cf3198ede1f
diff --git a/apps/media-server/src/__tests__/loom-importer/import.integration.test.ts b/apps/media-server/src/__tests__/loom-importer/import.integration.test.ts
new file mode 100644
index 000000000..b003dde6c
--- /dev/null
+++ b/apps/media-server/src/__tests__/loom-importer/import.integration.test.ts
@@ -0,0 +1,385 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { spawnSync } from "bun";
+import { createJob, generateJobId, getJob } from "../../lib/job-manager";
+import { Semaphore } from "../../loom-importer/fetch";
+import {
+ type ImporterSettings,
+ type ImportRequest,
+ runLoomImport,
+} from "../../loom-importer/job";
+
+const SIGNATURE = "Policy=signed&Signature=s&Key-Pair-Id=k";
+
+function ffmpeg(args: string[], cwd: string) {
+ const result = spawnSync({
+ cmd: ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", ...args],
+ cwd,
+ stderr: "pipe",
+ });
+ if (result.exitCode !== 0)
+ throw new Error(result.stderr.toString().slice(-2000));
+}
+
+async function ffmpegAsync(args: string[], cwd: string) {
+ const proc = Bun.spawn({
+ cmd: ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", ...args],
+ cwd,
+ stderr: "pipe",
+ });
+ const [stderr, exitCode] = await Promise.all([
+ new Response(proc.stderr).text(),
+ proc.exited,
+ ]);
+ if (exitCode !== 0) throw new Error(stderr.slice(-2000));
+}
+
+function probe(path: string) {
+ const result = spawnSync({
+ cmd: [
+ "ffprobe",
+ "-v",
+ "error",
+ "-show_entries",
+ "stream=codec_type,codec_name,width,start_time:format=duration",
+ "-of",
+ "json",
+ path,
+ ],
+ stderr: "pipe",
+ });
+ if (result.exitCode !== 0) throw new Error(result.stderr.toString());
+ return JSON.parse(result.stdout.toString()) as {
+ streams: Array<{
+ codec_type: string;
+ codec_name: string;
+ width?: number;
+ start_time: string;
+ }>;
+ format: { duration: string };
+ };
+}
+
+function streamStart(output: ReturnType<typeof probe>, type: string) {
+ return Number(
+ output.streams.find((stream) => stream.codec_type === type)?.start_time,
+ );
+}
+
+describe("runLoomImport", () => {
+ let root: string;
+ let server: ReturnType<typeof Bun.serve>;
+ let base: string;
+ const uploads = new Map<string, Uint8Array>();
+ const webhooks: Array<Record<string, unknown>> = [];
+ const sourceRequests: string[] = [];
+
+ beforeAll(async () => {
+ root = await mkdtemp(join(tmpdir(), "loom-importer-test-"));
+ const hls = join(root, "hls");
+ const dash = join(root, "dash");
+ await mkdir(hls);
+ await mkdir(dash);
+ ffmpeg(
+ [
+ "-f",
+ "lavfi",
+ "-i",
+ "testsrc2=size=320x240:rate=30",
+ "-itsoffset",
+ "0.25",
+ "-f",
+ "lavfi",
+ "-i",
+ "sine=frequency=440:sample_rate=48000",
+ "-t",
+ "9",
+ "-map",
+ "0:v",
+ "-map",
+ "0:v",
+ "-map",
+ "1:a",
+ "-c:v",
+ "libx264",
+ "-preset",
+ "ultrafast",
+ "-g",
+ "30",
+ "-s:v:1",
+ "160x120",
+ "-b:v:0",
+ "600k",
+ "-b:v:1",
+ "150k",
+ "-c:a",
+ "aac",
+ "-f",
+ "hls",
+ "-hls_time",
+ "2",
+ "-hls_playlist_type",
+ "vod",
+ "-master_pl_name",
+ "master.m3u8",
+ "-var_stream_map",
+ "v:0,agroup:aud v:1,agroup:aud a:0,agroup:aud,default:yes",
+ "-hls_segment_filename",
+ "stream%v-%03d.ts",
+ "stream%v.m3u8",
+ ],
+ hls,
+ );
+ ffmpeg(
+ [
+ "-f",
+ "lavfi",
+ "-i",
+ "testsrc2=size=320x240:rate=30",
+ "-f",
+ "lavfi",
+ "-i",
+ "sine=frequency=440:sample_rate=48000",
+ "-t",
+ "9",
+ "-c:v",
+ "libvpx-vp9",
+ "-deadline",
+ "realtime",
+ "-cpu-used",
+ "8",
+ "-b:v",
+ "300k",
+ "-g",
+ "60",
+ "-c:a",
+ "libopus",
+ "-f",
+ "dash",
+ "-seg_duration",
+ "2",
+ "-use_template",
+ "1",
+ "-use_timeline",
+ "1",
+ "-dash_segment_type",
+ "webm",
+ "manifest.mpd",
+ ],
+ dash,
+ );
+ server = Bun.serve({
+ port: 0,
+ hostname: "127.0.0.1",
+ async fetch(request) {
+ const url = new URL(request.url);
+ if (url.pathname.startsWith("/open/")) {
+ const file = Bun.file(join(root, url.pathname.slice(6)));
+ return (await file.exists())
+ ? new Response(file)
+ : new Response("missing", { status: 404 });
+ }
+ if (url.pathname.startsWith("/media/")) {
+ sourceRequests.push(url.pathname);
+ if (url.searchParams.get("Signature") !== "s")
+ return new Response("missing signature", { status: 403 });
+ const file = Bun.file(join(root, url.pathname.slice(7)));
+ return (await file.exists())
+ ? new Response(file)
+ : new Response("missing", { status: 404 });
+ }
+ if (url.pathname.startsWith("/put/") && request.method === "PUT") {
+ uploads.set(
+ url.pathname.slice(5),
+ new Uint8Array(await request.arrayBuffer()),
+ );
+ return new Response(null, {
+ status: 200,
+ headers: { ETag: '"etag"' },
+ });
+ }
+ if (url.pathname === "/webhook") {
+ webhooks.push(await request.json());
+ return Response.json({ success: true });
+ }
+ return new Response("not found", { status: 404 });
+ },
+ });
+ base = `http://127.0.0.1:${server.port}`;
+ }, 120_000);
+
+ afterAll(async () => {
+ server?.stop(true);
+ if (root) await rm(root, { recursive: true, force: true });
+ });
+
+ const settings = (): ImporterSettings => ({
+ workRoot: join(root, "work"),
+ allowedHosts: ["127.0.0.1"],
+ globalSemaphore: new Semaphore(64),
+ jobConnections: 8,
+ rangedConnections: 4,
+ segmentAttempts: 2,
+ segmentTimeoutMs: 10_000,
+ remuxTimeoutMs: 60_000,
+ terminalWebhookAttempts: 2,
+ minFreeBytes: 1,
+ });
+
+ const request = (
+ videoId: string,
+ videoUrl: string,
+ extra: Partial<ImportRequest> = {},
+ ): ImportRequest => ({
+ videoId,
+ userId: "user",
+ videoUrl,
+ outputPresignedUrl: `${base}/put/${videoId}/result.mp4`,
+ thumbnailPresignedUrl: `${base}/put/${videoId}/thumbnail.jpg`,
+ previewGifPresignedUrl: `${base}/put/${videoId}/preview.gif`,
+ webhookUrl: `${base}/webhook`,
+ ...extra,
+ });
+
+ async function run(input: ImportRequest) {
+ const job = createJob(
+ generateJobId(),
+ input.videoId,
+ input.userId,
+ input.webhookUrl,
+ );
+ await runLoomImport(
+ job,
+ input,
+ settings(),
+ new Semaphore(8),
+ new AbortController().signal,
+ );
+ return getJob(job.jobId);
+ }
+
+ test("imports the best HLS variant as a playable MP4 with unchanged timing", async () => {
+ const job = await run(
+ request("hls-video", `${base}/media/hls/master.m3u8?${SIGNATURE}`),
+ );
+ expect(job?.phase).toBe("complete");
+ expect(job?.deferredSourceKey).toBeUndefined();
+ expect(job?.metadata).toMatchObject({
+ width: 320,
+ height: 240,
+ fps: 30,
+ videoCodec: "h264",
+ audioCodec: "aac",
+ });
+ const uploaded = uploads.get("hls-video/result.mp4");
+ expect(uploaded?.byteLength).toBeGreaterThan(10_000);
+ expect(uploads.get("hls-video/thumbnail.jpg")?.byteLength).toBeGreaterThan(
+ 0,
+ );
+ expect(uploads.get("hls-video/preview.gif")?.byteLength).toBeGreaterThan(0);
+ const imported = join(root, "imported.mp4");
+ await writeFile(imported, uploaded ?? new Uint8Array());
+ const reference = join(root, "reference.mp4");
+ await ffmpegAsync(
+ [
+ "-protocol_whitelist",
+ "file,http,tcp",
+ "-i",
+ `${base}/open/hls/master.m3u8`,
+ "-map",
+ "0:v:0",
+ "-map",
+ "0:a:0",
+ "-c",
+ "copy",
+ reference,
+ ],
+ root,
+ );
+ const importedProbe = probe(imported);
+ const referenceProbe = probe(reference);
+ expect(
+ importedProbe.streams.find((s) => s.codec_type === "video")?.width,
+ ).toBe(320);
+ const offset = (output: ReturnType<typeof probe>) =>
+ streamStart(output, "audio") - streamStart(output, "video");
+ expect(offset(importedProbe)).toBeCloseTo(offset(referenceProbe), 3);
+ expect(offset(importedProbe)).toBeCloseTo(0.25, 1);
+ expect(Number(importedProbe.format.duration)).toBeCloseTo(
+ Number(referenceProbe.format.duration),
+ 0,
+ );
+ expect(sourceRequests.some((path) => path.includes("stream1-"))).toBe(
+ false,
+ );
+ const complete = webhooks.findLast(
+ (payload) => payload.videoId === "hls-video",
+ );
+ expect(complete?.phase).toBe("complete");
+ expect(complete?.metadata).toMatchObject({ width: 320, fps: 30 });
+ }, 60_000);
+
+ test("stores VP9 DASH as the original WebM and reports it for deferred processing", async () => {
+ const job = await run(
+ request("dash-video", `${base}/media/dash/manifest.mpd?${SIGNATURE}`, {
+ originalUpload: {
+ key: "user/dash-video/raw-upload.webm",
+ upload: { type: "put", url: `${base}/put/dash-video/raw.webm` },
+ },
+ }),
+ );
+ expect(job?.phase).toBe("complete");
+ expect(job?.deferredSourceKey).toBe("user/dash-video/raw-upload.webm");
+ expect(job?.metadata).toMatchObject({
+ videoCodec: "vp9",
+ audioCodec: "opus",
+ width: 320,
+ });
+ expect(job?.metadata?.duration).toBeCloseTo(9, 0);
+ expect(uploads.has("dash-video/result.mp4")).toBe(false);
+ const original = join(root, "original.webm");
+ await writeFile(
+ original,
+ uploads.get("dash-video/raw.webm") ?? new Uint8Array(),
+ );
+ const originalProbe = probe(original);
+ expect(originalProbe.streams.map((s) => s.codec_name).sort()).toEqual([
+ "opus",
+ "vp9",
+ ]);
+ const complete = webhooks.findLast(
+ (payload) => payload.videoId === "dash-video",
+ );
+ expect(complete?.deferredSourceKey).toBe("user/dash-video/raw-upload.webm");
+ }, 60_000);
+
+ test("fails VP9 sources that have nowhere to store the original", async () => {
+ const job = await run(
+ request("dash-no-target", `${base}/media/dash/manifest.mpd?${SIGNATURE}`),
+ );
+ expect(job?.phase).toBe("error");
+ expect(job?.error).toContain("original upload target");
+ }, 60_000);
+
+ test("fails without retrying forever when segments are not signed", async () => {
+ const job = await run(
+ request("unsigned", `${base}/media/hls/master.m3u8?Policy=x`),
+ );
+ expect(job?.phase).toBe("error");
+ expect(job?.error).toContain("403");
+ }, 60_000);
+
+ test("refuses sources outside the allowed hosts", async () => {
+ const job = await run(
+ request("foreign", "https://example.com/master.m3u8?Policy=x"),
+ );
+ expect(job?.phase).toBe("error");
+ expect(job?.error).toContain("not allowed");
+ });
+
+ test("leaves no working files behind", async () => {
+ expect(await readdir(join(root, "work"))).toEqual([]);
+ });
+});
diff --git a/apps/media-server/src/__tests__/loom-importer/parsing.test.ts b/apps/media-server/src/__tests__/loom-importer/parsing.test.ts
new file mode 100644
index 000000000..c91b4d938
--- /dev/null
+++ b/apps/media-server/src/__tests__/loom-importer/parsing.test.ts
@@ -0,0 +1,229 @@
+import { describe, expect, test } from "bun:test";
+import {
+ assertAllowedSourceUrl,
+ LoomSourceRejectedError,
+ parseAllowedHosts,
+ Semaphore,
+} from "../../loom-importer/fetch";
+import {
+ buildRemuxArgs,
+ chooseContainer,
+ resolveFps,
+} from "../../loom-importer/media";
+import {
+ listMediaPlaylistResources,
+ parseHlsAttributes,
+ selectHlsTracks,
+ UnsupportedStreamError,
+} from "../../loom-importer/streaming";
+
+describe("selectHlsTracks", () => {
+ const master = [
+ "#EXTM3U",
+ "#EXT-X-VERSION:3",
+ '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="low",NAME="low",DEFAULT=YES,URI="/m/audio-low.m3u8"',
+ '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="alt",DEFAULT=NO,URI="/m/audio-alt.m3u8"',
+ '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="main",DEFAULT=YES,URI="/m/audio-main.m3u8"',
+ '#EXT-X-STREAM-INF:BANDWIDTH=1500000,RESOLUTION=1280x720,CODECS="avc1.4d001f,mp4a.40.2",AUDIO="low"',
+ "/m/video-1500.m3u8",
+ '#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=9000000,URI="/m/iframes.m3u8"',
+ '#EXT-X-STREAM-INF:BANDWIDTH=5500000,RESOLUTION=2400x1350,CODECS="avc1.4d002a,mp4a.40.2",AUDIO="audio"',
+ "",
+ "/m/video-5500.m3u8",
+ '#EXT-X-STREAM-INF:BANDWIDTH=5500000,RESOLUTION=1920x1080,AUDIO="audio"',
+ "/m/video-5500-small.m3u8",
+ ].join("\n");
+
+ test("picks the highest bandwidth variant and its default audio rendition", () => {
+ const selected = selectHlsTracks(master);
+ expect(selected?.video).toBe("/m/video-5500.m3u8");
+ expect(selected?.audio).toBe("/m/audio-main.m3u8");
+ expect(selected?.masterLines).toEqual([
+ "#EXTM3U",
+ "#EXT-X-VERSION:3",
+ '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="main",DEFAULT=YES,URI="/m/audio-main.m3u8"',
+ '#EXT-X-STREAM-INF:BANDWIDTH=5500000,RESOLUTION=2400x1350,CODECS="avc1.4d002a,mp4a.40.2",AUDIO="audio"',
+ "/m/video-5500.m3u8",
+ ]);
+ });
+
+ test("keeps muxed audio when the variant has no audio group", () => {
+ const selected = selectHlsTracks(
+ "#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=10\nlow.m3u8\n#EXT-X-STREAM-INF:BANDWIDTH=20\nhigh.m3u8",
+ );
+ expect(selected?.video).toBe("high.m3u8");
+ expect(selected?.audio).toBeNull();
+ });
+
+ test("treats a media playlist as having no variants", () => {
+ expect(
+ selectHlsTracks("#EXTM3U\n#EXTINF:4,\nhttps://cdn.loom.com/a.ts"),
+ ).toBeNull();
+ });
+
+ test("parses quoted attribute values containing commas", () => {
+ expect(
+ parseHlsAttributes(
+ 'BANDWIDTH=10,CODECS="avc1.4d002a,mp4a.40.2",AUDIO="a"',
+ ),
+ ).toEqual({ BANDWIDTH: "10", CODECS: "avc1.4d002a,mp4a.40.2", AUDIO: "a" });
+ });
+});
+
+describe("listMediaPlaylistResources", () => {
+ test("lists the init map and every segment with their line positions", () => {
+ const resources = listMediaPlaylistResources(
+ [
+ "#EXTM3U",
+ '#EXT-X-MAP:URI="https://luna.loom.com/init.webm?Policy=x"',
+ "#EXTINF:4.0,",
+ "https://luna.loom.com/v-0.webm?Policy=x",
+ "#EXTINF:4.0,",
+ "https://luna.loom.com/v-1.webm?Policy=x",
+ "#EXT-X-ENDLIST",
+ ].join("\n"),
+ );
+ expect(resources).toEqual([
+ {
+ url: "https://luna.loom.com/init.webm?Policy=x",
+ lineIndex: 1,
+ attribute: true,
+ },
+ {
+ url: "https://luna.loom.com/v-0.webm?Policy=x",
+ lineIndex: 3,
+ attribute: false,
+ },
+ {
+ url: "https://luna.loom.com/v-1.webm?Policy=x",
+ lineIndex: 5,
+ attribute: false,
+ },
+ ]);
+ });
+
+ test.each([
+ '#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI="k"\n#EXTINF:4,\na.ts',
+ "#EXTM3U\n#EXT-X-BYTERANGE:100@0\n#EXTINF:4,\na.ts",
+ '#EXTM3U\n#EXT-X-MAP:URI="i.mp4",BYTERANGE="10@0"\n#EXTINF:4,\na.ts',
+ "#EXTM3U\n#EXT-X-ENDLIST",
+ ])("rejects playlists it cannot prefetch: %s", (playlist) => {
+ expect(() => listMediaPlaylistResources(playlist)).toThrow(
+ UnsupportedStreamError,
+ );
+ });
+
+ test("accepts an explicit unencrypted key tag", () => {
+ expect(
+ listMediaPlaylistResources(
+ "#EXTM3U\n#EXT-X-KEY:METHOD=NONE\n#EXTINF:4,\na.ts",
+ ),
+ ).toHaveLength(1);
+ });
+});
+
+describe("assertAllowedSourceUrl", () => {
+ const hosts = parseAllowedHosts(undefined);
+
+ test.each([
+ "https://luna.loom.com/id/x/playlist.m3u8?Policy=a",
+ "https://cdn.loom.com/sessions/transcoded/x.mp4",
+ "https://loom.com/x.ts",
+ ])("allows Loom hosts over https: %s", (url) => {
+ expect(assertAllowedSourceUrl(url, hosts).href).toBe(url);
+ });
+
+ test.each([
+ "http://luna.loom.com/a.ts",
+ "https://loom.com.attacker.example/a.ts",
+ "https://attackerloom.com/a.ts",
+ "http://169.254.169.254/latest/meta-data/",
+ "https://user:[email protected]/a.ts",
+ "file:///etc/passwd",
+ "not a url",
+ ])("rejects anything else: %s", (url) => {
+ expect(() => assertAllowedSourceUrl(url, hosts)).toThrow(
+ LoomSourceRejectedError,
+ );
+ });
+
+ test("allows plain http only for an exact configured host", () => {
+ const local = parseAllowedHosts("127.0.0.1");
+ expect(assertAllowedSourceUrl("http://127.0.0.1:9/a.ts", local).host).toBe(
+ "127.0.0.1:9",
+ );
+ expect(() =>
+ assertAllowedSourceUrl("http://localhost/a.ts", local),
+ ).toThrow(LoomSourceRejectedError);
+ });
+});
+
+describe("media decisions", () => {
+ test.each([
+ [{ video: "h264", audio: "aac" }, "mp4"],
+ [{ video: "h264", audio: null }, "mp4"],
+ [{ video: "vp9", audio: "opus" }, "webm"],
+ [{ video: "vp8", audio: "vorbis" }, "webm"],
+ [{ video: "h264", audio: "opus" }, null],
+ [{ video: "hevc", audio: "aac" }, null],
+ [{ video: null, audio: "aac" }, null],
+ ] as const)("chooses a container for %o", (codecs, expected) => {
+ expect(chooseContainer(codecs)).toBe(expected);
+ });
+
+ test("uses the measured packet rate when the average rate is missing", () => {
+ expect(
+ resolveFps(
+ {
+ avg_frame_rate: "0/0",
+ r_frame_rate: "1000/1",
+ nb_read_packets: "600",
+ },
+ 20,
+ ),
+ ).toBe(30);
+ });
+
+ test("rejects implausible nominal rates", () => {
+ expect(resolveFps({ r_frame_rate: "1000/1" }, 0)).toBeNull();
+ expect(resolveFps({ avg_frame_rate: "30000/1001" }, 10)).toBe(29.97);
+ });
+
+ test("maps only the first video and audio stream", () => {
+ const args = buildRemuxArgs([], "in.m3u8", "out.mp4", "mp4", true);
+ expect(args.join(" ")).toContain("-map 0:v:0 -map 0:a:0");
+ expect(args.join(" ")).toContain("-c copy -movflags +faststart -f mp4");
+ expect(
+ buildRemuxArgs([], "in.m3u8", "out.webm", "webm", false).join(" "),
+ ).not.toContain("0:a:0");
+ });
+});
+
+describe("Semaphore", () => {
+ test("never runs more than its size at once", async () => {
+ const semaphore = new Semaphore(3);
+ let active = 0;
+ let peak = 0;
+ await Promise.all(
+ Array.from({ length: 20 }, () =>
+ semaphore.run(async () => {
+ active++;
+ peak = Math.max(peak, active);
+ await new Promise((resolve) => setTimeout(resolve, 2));
+ active--;
+ }),
+ ),
+ );
+ expect(peak).toBe(3);
+ });
+
+ test("releases its slot when work throws", async () => {
+ const semaphore = new Semaphore(1);
+ await expect(
+ semaphore.run(async () => {
+ throw new Error("boom");
+ }),
+ ).rejects.toThrow("boom");
+ expect(await semaphore.run(async () => "ok")).toBe("ok");
+ });
+});
diff --git a/apps/media-server/src/lib/job-manager.ts b/apps/media-server/src/lib/job-manager.ts
index 5c05238f2..40934074e 100644
--- a/apps/media-server/src/lib/job-manager.ts
+++ b/apps/media-server/src/lib/job-manager.ts
@@ -77,6 +77,7 @@ export interface JobProgress {
errorCode?: RecordingErrorCode;
metadata?: VideoMetadata;
outputUrl?: string;
+ deferredSourceKey?: string;
}
export interface RecordingWorkerAcknowledgement {
@@ -137,6 +138,7 @@ export interface Job {
errorCode?: RecordingErrorCode;
metadata?: VideoMetadata;
outputUrl?: string;
+ deferredSourceKey?: string;
createdAt: number;
updatedAt: number;
inputTempFile?: TempFileHandle;
@@ -405,6 +407,7 @@ export function updateJob(
| "manifestSha256"
| "recordingVerification"
| "outputUrl"
+ | "deferredSourceKey"
| "inputTempFile"
| "outputTempFile"
| "mediaOperation"
@@ -689,6 +692,7 @@ export function getJobProgress(job: Job): JobProgress {
errorCode: job.errorCode,
metadata: job.metadata,
outputUrl: job.outputUrl,
+ deferredSourceKey: job.deferredSourceKey,
};
}
diff --git a/apps/media-server/src/loom-importer/app.ts b/apps/media-server/src/loom-importer/app.ts
new file mode 100644
index 000000000..709add741
--- /dev/null
+++ b/apps/media-server/src/loom-importer/app.ts
@@ -0,0 +1,211 @@
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { Hono } from "hono";
+import { validateMediaServerSecret } from "../lib/auth";
+import {
+ createJob,
+ deleteJob,
+ generateJobId,
+ getAllJobs,
+ getJob,
+ getJobProgress,
+ type Job,
+ sendWebhook,
+ updateJob,
+} from "../lib/job-manager";
+import { LOOM_IMPORT_MAX_BYTES } from "../lib/media-video";
+import { parseAllowedHosts, Semaphore } from "./fetch";
+import {
+ type ImporterSettings,
+ type ImportRequest,
+ importRequestSchema,
+ runLoomImport,
+} from "./job";
+
+function positiveInt(value: string | undefined, fallback: number): number {
+ const parsed = Number.parseInt(value ?? "", 10);
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+export function loadImporterSettings(
+ env: Record<string, string | undefined> = process.env,
+) {
+ const settings: ImporterSettings = {
+ workRoot: env.LOOM_IMPORTER_WORK_DIR ?? join(tmpdir(), "loom-importer"),
+ allowedHosts: parseAllowedHosts(env.LOOM_IMPORTER_ALLOWED_HOSTS),
+ globalSemaphore: new Semaphore(
+ positiveInt(env.LOOM_IMPORTER_TOTAL_CONNECTIONS, 384),
+ ),
+ jobConnections: positiveInt(env.LOOM_IMPORTER_JOB_CONNECTIONS, 48),
+ rangedConnections: Math.min(
+ 32,
+ positiveInt(env.LOOM_IMPORTER_RANGED_CONNECTIONS, 16),
+ ),
+ segmentAttempts: positiveInt(env.LOOM_IMPORTER_SEGMENT_ATTEMPTS, 6),
+ segmentTimeoutMs: positiveInt(env.LOOM_IMPORTER_SEGMENT_TIMEOUT_MS, 60_000),
+ remuxTimeoutMs: positiveInt(env.LOOM_IMPORTER_REMUX_TIMEOUT_MS, 1_800_000),
+ terminalWebhookAttempts: positiveInt(env.LOOM_IMPORTER_WEBHOOK_ATTEMPTS, 8),
+ minFreeBytes:
+ positiveInt(env.LOOM_IMPORTER_MIN_FREE_GB, 25) * 1024 * 1024 * 1024,
+ };
+ return {
+ settings,
+ maxActiveJobs: positiveInt(env.LOOM_IMPORTER_MAX_JOBS, 16),
+ maxQueuedJobs: positiveInt(env.LOOM_IMPORTER_MAX_QUEUED, 400),
+ };
+}
+
+const TERMINAL_JOB_TTL_MS = 24 * 60 * 60 * 1000;
+
+function isActive(job: Job) {
+ return (
+ job.phase !== "complete" &&
+ job.phase !== "error" &&
+ job.phase !== "cancelled"
+ );
+}
+
+export function createLoomImporter(
+ config: ReturnType<typeof loadImporterSettings> = loadImporterSettings(),
+) {
+ const queue: Array<{ job: Job; request: ImportRequest }> = [];
+ const running = new Map<string, AbortController>();
+ let draining = false;
+
+ const pump = () => {
+ while (running.size < config.maxActiveJobs && queue.length > 0) {
+ const next = queue.shift();
+ if (!next) break;
+ const controller = new AbortController();
+ running.set(next.job.jobId, controller);
+ void runLoomImport(
+ next.job,
+ next.request,
+ config.settings,
+ new Semaphore(config.settings.jobConnections),
+ controller.signal,
+ ).finally(() => {
+ running.delete(next.job.jobId);
+ setTimeout(
+ () => deleteJob(next.job.jobId),
+ TERMINAL_JOB_TTL_MS,
+ ).unref?.();
+ pump();
+ });
+ }
+ };
+
+ const app = new Hono().basePath("/loom-importer");
+
+ app.get("/health", (c) =>
+ c.json({
+ status: draining ? "draining" : "ok",
+ running: running.size,
+ queued: queue.length,
+ }),
+ );
+
+ app.get("/video/status", (c) => {
+ if (!validateMediaServerSecret(c))
+ return c.json({ error: "Unauthorized" }, 401);
+ return c.json({
+ loomImporter: true,
+ loomImportMaxBytes: LOOM_IMPORT_MAX_BYTES,
+ loomMultipartUploads: true,
+ loomDeferredOriginals: true,
+ draining,
+ running: running.size,
+ queued: queue.length,
+ maxActiveJobs: config.maxActiveJobs,
+ maxQueuedJobs: config.maxQueuedJobs,
+ jobs: getAllJobs()
+ .filter(isActive)
+ .map((job) => ({
+ jobId: job.jobId,
+ videoId: job.videoId,
+ phase: job.phase,
+ progress: job.progress,
+ ageSeconds: Math.round((Date.now() - job.createdAt) / 1000),
+ })),
+ });
+ });
+
+ app.post("/video/import", async (c) => {
+ if (!validateMediaServerSecret(c))
+ return c.json({ error: "Unauthorized" }, 401);
+ const parsed = importRequestSchema.safeParse(
+ await c.req.json().catch(() => null),
+ );
+ if (!parsed.success)
+ return c.json(
+ {
+ error: "Invalid request",
+ code: "INVALID_REQUEST",
+ details: parsed.error.message,
+ },
+ 400,
+ );
+ if (draining || queue.length >= config.maxQueuedJobs) {
+ c.header("Retry-After", "30");
+ return c.json(
+ {
+ error: "Server is busy",
+ code: "SERVER_BUSY",
+ details: draining
+ ? "Loom importer is draining"
+ : "Loom import queue is full",
+ },
+ 503,
+ );
+ }
+ const request = parsed.data;
+ const job = createJob(
+ generateJobId(),
+ request.videoId,
+ request.userId,
+ request.webhookUrl,
+ request.webhookSecret,
+ );
+ queue.push({ job, request });
+ pump();
+ return c.json({
+ jobId: job.jobId,
+ status: "queued",
+ message: "Loom import queued",
+ });
+ });
+
+ app.get("/video/process/:jobId/status", (c) => {
+ if (!validateMediaServerSecret(c))
+ return c.json({ error: "Unauthorized" }, 401);
+ const job = getJob(c.req.param("jobId"));
+ if (!job) return c.json({ error: "Job not found", code: "NOT_FOUND" }, 404);
+ return c.json(getJobProgress(job));
+ });
+
+ const cancelJob = async (job: Job) => {
+ const cancelled = updateJob(job.jobId, {
+ phase: "cancelled",
+ progress: 0,
+ message: "Server shutting down",
+ });
+ if (cancelled) await sendWebhook(cancelled);
+ };
+
+ const drain = async (timeoutMs: number) => {
+ draining = true;
+ const deadline = Date.now() + timeoutMs;
+ while ((queue.length > 0 || running.size > 0) && Date.now() < deadline)
+ await new Promise((resolve) => setTimeout(resolve, 500));
+ await Promise.all(queue.splice(0).map(({ job }) => cancelJob(job)));
+ for (const controller of running.values()) controller.abort();
+ while (running.size > 0)
+ await new Promise((resolve) => setTimeout(resolve, 250));
+ };
+
+ return {
+ app,
+ drain,
+ stats: () => ({ running: running.size, queued: queue.length }),
+ };
+}
diff --git a/apps/media-server/src/loom-importer/fetch.ts b/apps/media-server/src/loom-importer/fetch.ts
new file mode 100644
index 000000000..e01431feb
--- /dev/null
+++ b/apps/media-server/src/loom-importer/fetch.ts
@@ -0,0 +1,188 @@
+import { open, rename, rm } from "node:fs/promises";
+
+export class LoomSourceRejectedError extends Error {}
+
+export class Semaphore {
+ private available: number;
+ private readonly waiters: Array<() => void> = [];
+
+ constructor(size: number) {
+ this.available = Math.max(1, size);
+ }
+
+ async run<T>(work: () => Promise<T>): Promise<T> {
+ if (this.available > 0) {
+ this.available--;
+ } else {
+ await new Promise<void>((resolve) => this.waiters.push(resolve));
+ }
+ try {
+ return await work();
+ } finally {
+ const next = this.waiters.shift();
+ if (next) next();
+ else this.available++;
+ }
+ }
+}
+
+export function parseAllowedHosts(value: string | undefined): string[] {
+ const hosts = (value ?? "loom.com")
+ .split(",")
+ .map((host) => host.trim().toLowerCase())
+ .filter(Boolean);
+ return hosts.length > 0 ? hosts : ["loom.com"];
+}
+
+export function assertAllowedSourceUrl(
+ value: string,
+ allowedHosts: string[],
+): URL {
+ let url: URL;
+ try {
+ url = new URL(value);
+ } catch {
+ throw new LoomSourceRejectedError("Loom source URL is invalid");
+ }
+ const hostname = url.hostname.toLowerCase();
+ const allowed = allowedHosts.some(
+ (host) => hostname === host || hostname.endsWith(`.${host}`),
+ );
+ const secure =
+ url.protocol === "https:" ||
+ (url.protocol === "http:" && allowedHosts.includes(hostname));
+ if (!allowed || !secure || url.username || url.password) {
+ throw new LoomSourceRejectedError(
+ `Loom source host is not allowed: ${url.protocol}//${hostname}`,
+ );
+ }
+ return url;
+}
+
+export function createAllowedFetch(allowedHosts: string[]): typeof fetch {
+ const allowedFetch = (input: RequestInfo | URL, init?: RequestInit) => {
+ const url =
+ input instanceof Request
+ ? input.url
+ : input instanceof URL
+ ? input.href
+ : input;
+ assertAllowedSourceUrl(url, allowedHosts);
+ return fetch(input, { ...init, redirect: "error" });
+ };
+ return Object.assign(allowedFetch, { preconnect: fetch.preconnect });
+}
+
+const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]);
+const PERMANENT_STATUS_ATTEMPTS = 2;
+
+function retryDelayMs(attempt: number, retryAfter: string | null): number {
+ const seconds = Number(retryAfter);
+ if (Number.isFinite(seconds) && seconds > 0)
+ return Math.min(30_000, seconds * 1000);
+ return Math.min(8_000, 500 * 2 ** attempt) + Math.random() * 250;
+}
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+ return new Promise((resolve, reject) => {
+ if (signal?.aborted) return reject(signal.reason);
+ const timer = setTimeout(resolve, ms);
+ signal?.addEventListener(
+ "abort",
+ () => {
+ clearTimeout(timer);
+ reject(signal.reason);
+ },
+ { once: true },
+ );
+ });
+}
+
+export interface DownloadToFileOptions {
+ fetchImpl: typeof fetch;
+ signal: AbortSignal;
+ attempts: number;
+ requestTimeoutMs: number;
+}
+
+export async function downloadToFile(
+ url: string,
+ path: string,
+ options: DownloadToFileOptions,
+): Promise<number> {
+ let lastError: unknown;
+ let permanentFailures = 0;
+ for (let attempt = 0; attempt < options.attempts; attempt++) {
+ options.signal.throwIfAborted();
+ const partialPath = `${path}.part`;
+ try {
+ const response = await options.fetchImpl(url, {
+ signal: AbortSignal.any([
+ options.signal,
+ AbortSignal.timeout(options.requestTimeoutMs),
+ ]),
+ });
+ if (!response.ok) {
+ await response.body?.cancel().catch(() => {});
+ const error = new Error(
+ `Loom segment request failed with ${response.status}`,
+ );
+ if (!RETRYABLE_STATUS.has(response.status)) {
+ permanentFailures++;
+ if (permanentFailures >= PERMANENT_STATUS_ATTEMPTS) throw error;
+ }
+ lastError = error;
+ await sleep(
+ retryDelayMs(attempt, response.headers.get("retry-after")),
+ options.signal,
+ );
+ continue;
+ }
+ const expected = Number(response.headers.get("content-length"));
+ const bytes = await writeBody(response, partialPath);
+ if (Number.isFinite(expected) && expected > 0 && bytes !== expected) {
+ throw new Error(
+ `Loom segment was truncated (${bytes} of ${expected} bytes)`,
+ );
+ }
+ await rename(partialPath, path);
+ return bytes;
+ } catch (error) {
+ await rm(partialPath, { force: true }).catch(() => {});
+ if (options.signal.aborted) throw options.signal.reason ?? error;
+ if (
+ error instanceof LoomSourceRejectedError ||
+ permanentFailures >= PERMANENT_STATUS_ATTEMPTS
+ )
+ throw error;
+ lastError = error;
+ await sleep(retryDelayMs(attempt, null), options.signal);
+ }
+ }
+ throw lastError instanceof Error
+ ? lastError
+ : new Error("Loom segment download failed");
+}
+
+async function writeBody(response: Response, path: string): Promise<number> {
+ const handle = await open(path, "w");
+ let bytes = 0;
+ try {
+ if (!response.body) return 0;
+ for await (const chunk of response.body) {
+ let written = 0;
+ while (written < chunk.byteLength) {
+ const result = await handle.write(
+ chunk,
+ written,
+ chunk.byteLength - written,
+ );
+ written += result.bytesWritten;
+ }
+ bytes += chunk.byteLength;
+ }
+ return bytes;
+ } finally {
+ await handle.close();
+ }
+}
diff --git a/apps/media-server/src/loom-importer/index.ts b/apps/media-server/src/loom-importer/index.ts
new file mode 100644
index 000000000..cb2d9faaf
--- /dev/null
+++ b/apps/media-server/src/loom-importer/index.ts
@@ -0,0 +1,30 @@
+import { logger } from "hono/logger";
+import { getIdleTimeoutSeconds } from "../lib/lifecycle";
+import { createLoomImporter } from "./app";
+
+const port = Number(process.env.PORT) || 3456;
+const drainTimeoutMs =
+ Number(process.env.LOOM_IMPORTER_DRAIN_TIMEOUT_MS) || 10 * 60 * 1000;
+const importer = createLoomImporter();
+
+importer.app.use("*", logger());
+
+let stopping = false;
+const shutdown = async (signal: string) => {
+ if (stopping) process.exit(1);
+ stopping = true;
+ console.log(`[loom-importer] ${signal}: draining`, importer.stats());
+ await importer.drain(drainTimeoutMs);
+ process.exit(0);
+};
+
+for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const)
+ process.on(signal, () => void shutdown(signal));
+
+console.log(`[loom-importer] Starting on port ${port}`);
+
+export default {
+ port,
+ idleTimeout: getIdleTimeoutSeconds(),
+ fetch: importer.app.fetch,
+};
diff --git a/apps/media-server/src/loom-importer/job.ts b/apps/media-server/src/loom-importer/job.ts
new file mode 100644
index 000000000..ad6a1fedc
--- /dev/null
+++ b/apps/media-server/src/loom-importer/job.ts
@@ -0,0 +1,435 @@
+import { mkdir, mkdtemp, rm, statfs } from "node:fs/promises";
+import { join } from "node:path";
+import { z } from "zod";
+import {
+ type Job,
+ sendWebhook,
+ updateJob,
+ type VideoMetadata,
+} from "../lib/job-manager";
+import {
+ abortStorageUploadTarget,
+ generatePreviewGif,
+ generateThumbnail,
+ getFfmpegHlsCapabilities,
+ LOOM_IMPORT_MAX_BYTES,
+ materializeStreamingInput,
+ type StorageUploadTarget,
+ uploadFileToS3,
+ uploadFileToStorage,
+ uploadToS3,
+} from "../lib/media-video";
+import { downloadUrlToPathParallel } from "../lib/parallel-download";
+import type { TempFileHandle } from "../lib/temp-files";
+import {
+ assertAllowedSourceUrl,
+ createAllowedFetch,
+ type Semaphore,
+} from "./fetch";
+import {
+ buildRemuxArgs,
+ chooseContainer,
+ PLAYLIST_INPUT_ARGS,
+ probeCodecs,
+ probeOutput,
+ runCommand,
+} from "./media";
+import { prefetchMaterializedHls, UnsupportedStreamError } from "./streaming";
+
+const storageUploadSchema = z.discriminatedUnion("type", [
+ z.object({
+ type: z.literal("put"),
+ url: z.string().url(),
+ ifNoneMatch: z.literal("*").optional(),
+ }),
+ z.object({
+ type: z.literal("multipart"),
+ videoId: z.string(),
+ key: z.string().min(1),
+ uploadId: z.string().min(1),
+ partSize: z
+ .number()
+ .int()
+ .min(5 * 1024 * 1024),
+ signPartUrl: z.string().url(),
+ completeUrl: z.string().url(),
+ abortUrl: z.string().url(),
+ webhookSecret: z.string().optional(),
+ loomImportAttemptId: z.string().min(1).max(200).optional(),
+ }),
+]);
+
+export const importRequestSchema = z.object({
+ videoId: z.string().min(1),
+ userId: z.string().min(1),
+ videoUrl: z.string().url(),
+ outputPresignedUrl: z.string().url(),
+ outputUpload: storageUploadSchema.optional(),
+ sourceUpload: storageUploadSchema.optional(),
+ originalUpload: z
+ .object({
+ key: z.string().min(1).endsWith(".webm"),
+ upload: storageUploadSchema,
+ })
+ .optional(),
+ existingOutput: z
+ .object({
+ url: z.string().url(),
+ objectIdentity: z.string().regex(/^"[^"\r\n]+"$/),
+ fileSize: z.number().int().positive().max(LOOM_IMPORT_MAX_BYTES),
+ })
+ .optional(),
+ maxImportBytes: z
+ .number()
+ .int()
+ .positive()
+ .max(LOOM_IMPORT_MAX_BYTES)
+ .optional(),
+ thumbnailPresignedUrl: z.string().url().optional(),
+ previewGifPresignedUrl: z.string().url().optional(),
+ webhookUrl: z.string().url().optional(),
+ webhookSecret: z.string().optional(),
+});
+
+export type ImportRequest = z.infer<typeof importRequestSchema>;
+
+export interface ImporterSettings {
+ workRoot: string;
+ allowedHosts: string[];
+ globalSemaphore: Semaphore;
+ jobConnections: number;
+ rangedConnections: number;
+ segmentAttempts: number;
+ segmentTimeoutMs: number;
+ remuxTimeoutMs: number;
+ terminalWebhookAttempts: number;
+ minFreeBytes: number;
+}
+
+function isStreamingUrl(url: string): boolean {
+ const path = (url.split("?")[0] ?? "").toLowerCase();
+ return path.endsWith(".m3u8") || path.endsWith(".mpd");
+}
+
+function withSizeLimit(fetchImpl: typeof fetch, maxBytes: number) {
+ const limited = async (input: RequestInfo | URL, init?: RequestInit) => {
+ const response = await fetchImpl(input, init);
+ const total = Number(
+ response.headers.get("content-range")?.split("/")[1] ??
+ response.headers.get("content-length"),
+ );
+ if (Number.isFinite(total) && total > maxBytes) {
+ await response.body?.cancel().catch(() => {});
+ throw new Error("Loom imports are limited to 10 GB");
+ }
+ return response;
+ };
+ return Object.assign(limited, { preconnect: fetch.preconnect });
+}
+
+async function deliverTerminalWebhook(job: Job, attempts: number) {
+ for (let attempt = 0; attempt < attempts; attempt++) {
+ await sendWebhook(job);
+ if (job.terminalWebhookAcknowledgedAt || !job.webhookUrl) return;
+ await new Promise((resolve) =>
+ setTimeout(resolve, Math.min(60_000, 2_000 * 2 ** attempt)),
+ );
+ }
+ console.error(
+ `[loom-importer] Terminal webhook was not acknowledged for ${job.jobId}`,
+ );
+}
+
+function progressUpdate(
+ jobId: string,
+ phase: Job["phase"],
+ progress: number,
+ message: string,
+) {
+ const job = updateJob(jobId, { phase, progress, message });
+ if (job) void sendWebhook(job);
+}
+
+interface PreparedSource {
+ input: string;
+ inputArgs: string[];
+}
+
+async function prepareSource(
+ job: Job,
+ request: ImportRequest,
+ workDir: string,
+ settings: ImporterSettings,
+ jobSemaphore: Semaphore,
+ signal: AbortSignal,
+): Promise<PreparedSource> {
+ const maxBytes = request.maxImportBytes ?? LOOM_IMPORT_MAX_BYTES;
+ const allowedFetch = createAllowedFetch(settings.allowedHosts);
+ assertAllowedSourceUrl(request.videoUrl, settings.allowedHosts);
+ if (!isStreamingUrl(request.videoUrl)) {
+ const sourcePath = join(workDir, "source.mp4");
+ const result = await downloadUrlToPathParallel(
+ request.videoUrl,
+ sourcePath,
+ {
+ concurrency: settings.rangedConnections,
+ partBytes: 8 * 1024 * 1024,
+ signal,
+ fetchImpl: withSizeLimit(allowedFetch, maxBytes),
+ },
+ );
+ if (result.bytes > maxBytes)
+ throw new Error("Loom imports are limited to 10 GB");
+ return { input: sourcePath, inputArgs: [] };
+ }
+ const manifestDir = join(workDir, "manifest");
+ const segmentDir = join(workDir, "segments");
+ await mkdir(manifestDir);
+ await mkdir(segmentDir);
+ const master = await materializeStreamingInput(
+ request.videoUrl,
+ manifestDir,
+ signal,
+ );
+ if (!master.endsWith(".m3u8"))
+ throw new UnsupportedStreamError("This Loom DASH layout is not supported");
+ const prefetched = await prefetchMaterializedHls(master, segmentDir, {
+ fetchImpl: allowedFetch,
+ signal,
+ attempts: settings.segmentAttempts,
+ requestTimeoutMs: settings.segmentTimeoutMs,
+ jobSemaphore,
+ globalSemaphore: settings.globalSemaphore,
+ maxBytes,
+ onProgress: (progress) => {
+ if (
+ progress.completedSegments % 50 === 0 ||
+ progress.completedSegments === progress.totalSegments
+ )
+ progressUpdate(
+ job.jobId,
+ "downloading",
+ Math.round(
+ (progress.completedSegments / progress.totalSegments) * 60,
+ ),
+ `Downloading from Loom (${progress.completedSegments}/${progress.totalSegments})...`,
+ );
+ },
+ });
+ return {
+ input: prefetched.masterPath,
+ inputArgs: PLAYLIST_INPUT_ARGS(await getFfmpegHlsCapabilities()),
+ };
+}
+
+async function uploadPreviews(
+ outputPath: string,
+ metadata: VideoMetadata,
+ request: ImportRequest,
+ signal: AbortSignal,
+) {
+ const thumbnail = request.thumbnailPresignedUrl
+ ? generateThumbnail(outputPath, metadata.duration, {}, signal)
+ : undefined;
+ const gif: Promise<TempFileHandle> | undefined =
+ request.previewGifPresignedUrl
+ ? generatePreviewGif(outputPath, metadata.duration, {}, signal)
+ : undefined;
+ thumbnail?.catch(() => {});
+ gif?.catch(() => {});
+ try {
+ if (thumbnail && request.thumbnailPresignedUrl) {
+ const data = await thumbnail;
+ await uploadToS3(
+ data,
+ request.thumbnailPresignedUrl,
+ "image/jpeg",
+ signal,
+ );
+ }
+ } catch (error) {
+ signal.throwIfAborted();
+ console.warn("[loom-importer] Thumbnail failed", error);
+ }
+ try {
+ if (gif && request.previewGifPresignedUrl) {
+ const rendered = await gif;
+ try {
+ await uploadFileToS3(
+ rendered.path,
+ request.previewGifPresignedUrl,
+ "image/gif",
+ signal,
+ );
+ } finally {
+ await rendered.cleanup();
+ }
+ }
+ } catch (error) {
+ signal.throwIfAborted();
+ console.warn("[loom-importer] Preview GIF failed", error);
+ }
+}
+
+async function waitForDiskSpace(
+ path: string,
+ minFreeBytes: number,
+ signal: AbortSignal,
+) {
+ while (true) {
+ signal.throwIfAborted();
+ const stats = await statfs(path).catch(() => null);
+ if (!stats || stats.bavail * stats.bsize >= minFreeBytes) return;
+ await new Promise((resolve) => setTimeout(resolve, 5_000));
+ }
+}
+
+export async function runLoomImport(
+ job: Job,
+ request: ImportRequest,
+ settings: ImporterSettings,
+ jobSemaphore: Semaphore,
+ signal: AbortSignal,
+): Promise<void> {
+ const used = new Set<StorageUploadTarget>();
+ const targets = [
+ request.outputUpload,
+ request.sourceUpload,
+ request.originalUpload?.upload,
+ ].filter((target): target is StorageUploadTarget => target !== undefined);
+ let workDir: string | undefined;
+ try {
+ await mkdir(settings.workRoot, { recursive: true });
+ await waitForDiskSpace(settings.workRoot, settings.minFreeBytes, signal);
+ workDir = await mkdtemp(join(settings.workRoot, `${request.videoId}-`));
+ progressUpdate(job.jobId, "downloading", 0, "Downloading from Loom...");
+ let outputPath: string;
+ let deferredSourceKey: string | undefined;
+ if (request.existingOutput) {
+ outputPath = join(workDir, "result.mp4");
+ await downloadUrlToPathParallel(request.existingOutput.url, outputPath, {
+ concurrency: settings.rangedConnections,
+ partBytes: 8 * 1024 * 1024,
+ signal,
+ });
+ } else {
+ const source = await prepareSource(
+ job,
+ request,
+ workDir,
+ settings,
+ jobSemaphore,
+ signal,
+ );
+ const codecs = await probeCodecs(source.inputArgs, source.input, signal);
+ const container = chooseContainer(codecs);
+ if (!container)
+ throw new UnsupportedStreamError(
+ `Loom source codecs are not supported (${codecs.video ?? "none"}/${codecs.audio ?? "none"})`,
+ );
+ if (container === "webm" && !request.originalUpload)
+ throw new UnsupportedStreamError(
+ "This Loom recording needs an original upload target",
+ );
+ progressUpdate(job.jobId, "processing", 65, "Packaging video...");
+ outputPath = join(
+ workDir,
+ container === "mp4" ? "result.mp4" : "original.webm",
+ );
+ await runCommand(
+ buildRemuxArgs(
+ source.inputArgs,
+ source.input,
+ outputPath,
+ container,
+ codecs.audio !== null,
+ ),
+ settings.remuxTimeoutMs,
+ signal,
+ );
+ if (container === "webm") deferredSourceKey = request.originalUpload?.key;
+ }
+ const metadata = await probeOutput(outputPath, signal);
+ if (metadata.fileSize > (request.maxImportBytes ?? LOOM_IMPORT_MAX_BYTES))
+ throw new Error("Loom imports are limited to 10 GB");
+ updateJob(job.jobId, { metadata });
+ progressUpdate(job.jobId, "uploading", 75, "Saving video...");
+ if (!request.existingOutput) {
+ if (deferredSourceKey && request.originalUpload) {
+ used.add(request.originalUpload.upload);
+ await uploadFileToStorage(
+ outputPath,
+ request.originalUpload.upload,
+ "video/webm",
+ signal,
+ );
+ } else if (request.outputUpload) {
+ used.add(request.outputUpload);
+ await uploadFileToStorage(
+ outputPath,
+ request.outputUpload,
+ "video/mp4",
+ signal,
+ );
+ } else {
+ await uploadFileToS3(
+ outputPath,
+ request.outputPresignedUrl,
+ "video/mp4",
+ signal,
+ );
+ }
+ }
+ await Promise.all(
+ targets
+ .filter((target) => !used.has(target))
+ .map((target) => {
+ used.add(target);
+ return abortStorageUploadTarget(target).catch((error) =>
+ console.warn("[loom-importer] Upload cleanup failed", error),
+ );
+ }),
+ );
+ progressUpdate(
+ job.jobId,
+ "generating_thumbnail",
+ 90,
+ "Generating preview assets...",
+ );
+ await uploadPreviews(outputPath, metadata, request, signal);
+ signal.throwIfAborted();
+ const completed = updateJob(job.jobId, {
+ phase: "complete",
+ progress: 100,
+ message: deferredSourceKey
+ ? "Imported original Loom video"
+ : "Loom video imported",
+ deferredSourceKey,
+ });
+ if (completed)
+ await deliverTerminalWebhook(completed, settings.terminalWebhookAttempts);
+ } catch (error) {
+ await Promise.all(
+ targets
+ .filter((target) => !used.has(target))
+ .map((target) => abortStorageUploadTarget(target).catch(() => {})),
+ );
+ const cancelled = signal.aborted;
+ if (!cancelled) console.error(`[loom-importer] ${job.jobId} failed`, error);
+ const failed = updateJob(job.jobId, {
+ phase: cancelled ? "cancelled" : "error",
+ progress: 0,
+ error: cancelled
+ ? undefined
+ : error instanceof Error
+ ? error.message
+ : String(error),
+ message: cancelled ? "Server shutting down" : "Loom import failed",
+ });
+ if (failed)
+ await deliverTerminalWebhook(failed, settings.terminalWebhookAttempts);
+ } finally {
+ if (workDir)
+ await rm(workDir, { recursive: true, force: true }).catch(() => {});
+ }
+}
diff --git a/apps/media-server/src/loom-importer/media.ts b/apps/media-server/src/loom-importer/media.ts
new file mode 100644
index 000000000..93f822718
--- /dev/null
+++ b/apps/media-server/src/loom-importer/media.ts
@@ -0,0 +1,233 @@
+import { stat } from "node:fs/promises";
+import { spawn } from "bun";
+import type { VideoMetadata } from "../lib/job-manager";
+import type { FfmpegHlsCapabilities } from "../lib/media-video";
+import {
+ registerSubprocess,
+ terminateProcess,
+ unregisterSubprocess,
+} from "../lib/subprocess";
+
+const STDERR_LIMIT_BYTES = 64 * 1024;
+
+export const PLAYLIST_INPUT_ARGS = (capabilities: FfmpegHlsCapabilities) => [
+ "-protocol_whitelist",
+ "file,crypto,data",
+ "-allowed_extensions",
+ "ALL",
+ ...(capabilities.allowedSegmentExtensions
+ ? ["-allowed_segment_extensions", "ALL"]
+ : []),
+ ...(capabilities.extensionPicky ? ["-extension_picky", "0"] : []),
+];
+
+export async function runCommand(
+ cmd: string[],
+ timeoutMs: number,
+ signal: AbortSignal,
+): Promise<string> {
+ signal.throwIfAborted();
+ const proc = registerSubprocess(
+ spawn({ cmd, stdin: "ignore", stdout: "pipe", stderr: "pipe" }),
+ );
+ const stop = () => {
+ void terminateProcess(proc);
+ };
+ signal.addEventListener("abort", stop, { once: true });
+ const timer = setTimeout(stop, timeoutMs);
+ try {
+ const [stdout, stderr, exitCode] = await Promise.all([
+ new Response(proc.stdout).text(),
+ new Response(proc.stderr).text(),
+ proc.exited,
+ ]);
+ signal.throwIfAborted();
+ if (exitCode !== 0) {
+ throw new Error(
+ `${cmd[0]} exited with code ${exitCode}: ${stderr.slice(-STDERR_LIMIT_BYTES).trim().slice(-600)}`,
+ );
+ }
+ return stdout;
+ } finally {
+ clearTimeout(timer);
+ signal.removeEventListener("abort", stop);
+ unregisterSubprocess(proc);
+ }
+}
+
+interface ProbeStream {
+ codec_type?: string;
+ codec_name?: string;
+ width?: number;
+ height?: number;
+ avg_frame_rate?: string;
+ r_frame_rate?: string;
+ nb_read_packets?: string;
+ channels?: number;
+ sample_rate?: string;
+}
+
+interface ProbeOutput {
+ streams?: ProbeStream[];
+ format?: { duration?: string; bit_rate?: string };
+}
+
+export interface SourceCodecs {
+ video: string | null;
+ audio: string | null;
+}
+
+export async function probeCodecs(
+ inputArgs: string[],
+ input: string,
+ signal: AbortSignal,
+): Promise<SourceCodecs> {
+ const output = JSON.parse(
+ await runCommand(
+ [
+ "ffprobe",
+ "-v",
+ "error",
+ ...inputArgs,
+ "-show_entries",
+ "stream=codec_type,codec_name",
+ "-of",
+ "json",
+ input,
+ ],
+ 120_000,
+ signal,
+ ),
+ ) as ProbeOutput;
+ const streams = output.streams ?? [];
+ return {
+ video:
+ streams.find((stream) => stream.codec_type === "video")?.codec_name ??
+ null,
+ audio:
+ streams.find((stream) => stream.codec_type === "audio")?.codec_name ??
+ null,
+ };
+}
+
+function parseRate(value: string | undefined): number {
+ if (!value) return 0;
+ const [numerator, denominator] = value.split("/").map(Number);
+ if (!numerator || !denominator) return 0;
+ return numerator / denominator;
+}
+
+export function resolveFps(
+ stream: ProbeStream,
+ duration: number,
+): number | null {
+ const average = parseRate(stream.avg_frame_rate);
+ if (average > 0 && average <= 240) return Math.round(average * 100) / 100;
+ const packets = Number(stream.nb_read_packets);
+ if (packets > 0 && duration > 0) {
+ const measured = packets / duration;
+ if (measured > 0 && measured <= 240)
+ return Math.round(measured * 100) / 100;
+ }
+ const nominal = parseRate(stream.r_frame_rate);
+ if (nominal > 0 && nominal <= 240) return Math.round(nominal * 100) / 100;
+ return null;
+}
+
+export async function probeOutput(
+ path: string,
+ signal: AbortSignal,
+): Promise<VideoMetadata> {
+ const output = JSON.parse(
+ await runCommand(
+ [
+ "ffprobe",
+ "-v",
+ "error",
+ "-count_packets",
+ "-show_entries",
+ "format=duration,bit_rate:stream=codec_type,codec_name,width,height,avg_frame_rate,r_frame_rate,nb_read_packets,channels,sample_rate",
+ "-of",
+ "json",
+ path,
+ ],
+ 10 * 60_000,
+ signal,
+ ),
+ ) as ProbeOutput;
+ const streams = output.streams ?? [];
+ const video = streams.find((stream) => stream.codec_type === "video");
+ const audio = streams.find((stream) => stream.codec_type === "audio");
+ const duration = Number(output.format?.duration);
+ if (!video?.codec_name || !video.width || !video.height)
+ throw new Error("Imported Loom video has no readable video stream");
+ if (!Number.isFinite(duration) || duration <= 0)
+ throw new Error("Imported Loom video has no readable duration");
+ const fps = resolveFps(video, duration);
+ if (!fps) throw new Error("Imported Loom video has no readable frame rate");
+ const fileSize = (await stat(path)).size;
+ return {
+ duration,
+ width: video.width,
+ height: video.height,
+ fps,
+ videoCodec: video.codec_name,
+ audioCodec: audio?.codec_name ?? null,
+ audioChannels: audio?.channels ?? null,
+ sampleRate: audio?.sample_rate ? Number(audio.sample_rate) : null,
+ bitrate:
+ Number(output.format?.bit_rate) || Math.round((fileSize * 8) / duration),
+ fileSize,
+ };
+}
+
+export type ImportContainer = "mp4" | "webm";
+
+export function chooseContainer(codecs: SourceCodecs): ImportContainer | null {
+ if (
+ codecs.video === "h264" &&
+ (codecs.audio === null || codecs.audio === "aac")
+ )
+ return "mp4";
+ if (
+ (codecs.video === "vp9" ||
+ codecs.video === "vp8" ||
+ codecs.video === "av1") &&
+ (codecs.audio === null ||
+ codecs.audio === "opus" ||
+ codecs.audio === "vorbis")
+ )
+ return "webm";
+ return null;
+}
+
+export function buildRemuxArgs(
+ inputArgs: string[],
+ input: string,
+ output: string,
+ container: ImportContainer,
+ hasAudio: boolean,
+): string[] {
+ return [
+ "ffmpeg",
+ "-hide_banner",
+ "-nostdin",
+ "-loglevel",
+ "error",
+ ...inputArgs,
+ "-i",
+ input,
+ "-map",
+ "0:v:0",
+ ...(hasAudio ? ["-map", "0:a:0"] : []),
+ "-dn",
+ "-sn",
+ "-c",
+ "copy",
+ ...(container === "mp4"
+ ? ["-movflags", "+faststart", "-f", "mp4"]
+ : ["-f", "webm"]),
+ "-y",
+ output,
+ ];
+}
diff --git a/apps/media-server/src/loom-importer/streaming.ts b/apps/media-server/src/loom-importer/streaming.ts
new file mode 100644
index 000000000..06ddd19c8
--- /dev/null
+++ b/apps/media-server/src/loom-importer/streaming.ts
@@ -0,0 +1,304 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { extname, join } from "node:path";
+import {
+ type DownloadToFileOptions,
+ downloadToFile,
+ type Semaphore,
+} from "./fetch";
+
+export class UnsupportedStreamError extends Error {}
+
+export interface HlsVariant {
+ bandwidth: number;
+ pixels: number;
+ audioGroup: string | null;
+ uri: string;
+ line: string;
+}
+
+export interface HlsAudioRendition {
+ groupId: string;
+ isDefault: boolean;
+ uri: string | null;
+ line: string;
+}
+
+export interface SelectedTracks {
+ video: string;
+ audio: string | null;
+ masterLines: string[];
+}
+
+export function parseHlsAttributes(source: string): Record<string, string> {
+ const attributes: Record<string, string> = {};
+ const pattern = /([A-Z0-9-]+)=("[^"]*"|[^,]*)/g;
+ for (const match of source.matchAll(pattern)) {
+ const [, key, raw] = match;
+ if (!key || raw === undefined) continue;
+ attributes[key] =
+ raw.startsWith('"') && raw.endsWith('"') ? raw.slice(1, -1) : raw;
+ }
+ return attributes;
+}
+
+function tagValue(line: string): string {
+ return line.slice(line.indexOf(":") + 1);
+}
+
+export function selectHlsTracks(master: string): SelectedTracks | null {
+ const lines = master.split(/\r?\n/);
+ const variants: HlsVariant[] = [];
+ const audio: HlsAudioRendition[] = [];
+ for (let index = 0; index < lines.length; index++) {
+ const line = lines[index]?.trim() ?? "";
+ if (line.startsWith("#EXT-X-STREAM-INF:")) {
+ const attributes = parseHlsAttributes(tagValue(line));
+ let uriIndex = index + 1;
+ while (
+ uriIndex < lines.length &&
+ (!lines[uriIndex]?.trim() || lines[uriIndex]?.trim().startsWith("#"))
+ )
+ uriIndex++;
+ const uri = lines[uriIndex]?.trim();
+ if (!uri) continue;
+ const [width, height] = (attributes.RESOLUTION ?? "0x0")
+ .split("x")
+ .map(Number);
+ variants.push({
+ bandwidth: Number(attributes.BANDWIDTH) || 0,
+ pixels: (width || 0) * (height || 0),
+ audioGroup: attributes.AUDIO ?? null,
+ uri,
+ line,
+ });
+ index = uriIndex;
+ } else if (line.startsWith("#EXT-X-MEDIA:")) {
+ const attributes = parseHlsAttributes(tagValue(line));
+ if (attributes.TYPE !== "AUDIO" || !attributes["GROUP-ID"]) continue;
+ audio.push({
+ groupId: attributes["GROUP-ID"],
+ isDefault: attributes.DEFAULT === "YES",
+ uri: attributes.URI ?? null,
+ line,
+ });
+ }
+ }
+ if (variants.length === 0) return null;
+ const best = variants.reduce((current, candidate) =>
+ candidate.bandwidth > current.bandwidth ||
+ (candidate.bandwidth === current.bandwidth &&
+ candidate.pixels > current.pixels)
+ ? candidate
+ : current,
+ );
+ const group = best.audioGroup
+ ? audio.filter((rendition) => rendition.groupId === best.audioGroup)
+ : [];
+ const rendition =
+ group.find((candidate) => candidate.isDefault && candidate.uri) ??
+ group.find((candidate) => candidate.uri) ??
+ null;
+ const header = lines
+ .map((line) => line.trim())
+ .filter(
+ (line) =>
+ line === "#EXTM3U" ||
+ line.startsWith("#EXT-X-VERSION") ||
+ line.startsWith("#EXT-X-INDEPENDENT-SEGMENTS"),
+ );
+ return {
+ video: best.uri,
+ audio: rendition?.uri ?? null,
+ masterLines: [
+ ...(header.length > 0 ? header : ["#EXTM3U"]),
+ ...(rendition ? [rendition.line] : []),
+ best.line,
+ best.uri,
+ ],
+ };
+}
+
+export interface MediaPlaylistResource {
+ url: string;
+ lineIndex: number;
+ attribute: boolean;
+}
+
+export function listMediaPlaylistResources(
+ playlist: string,
+): MediaPlaylistResource[] {
+ const lines = playlist.split(/\r?\n/);
+ const resources: MediaPlaylistResource[] = [];
+ for (const [lineIndex, raw] of lines.entries()) {
+ const line = raw.trim();
+ if (!line) continue;
+ if (line.startsWith("#EXT-X-KEY:")) {
+ const method = parseHlsAttributes(tagValue(line)).METHOD;
+ if (method && method !== "NONE")
+ throw new UnsupportedStreamError("Encrypted HLS is not supported");
+ continue;
+ }
+ if (line.startsWith("#EXT-X-BYTERANGE"))
+ throw new UnsupportedStreamError("HLS byte ranges are not supported");
+ if (line.startsWith("#EXT-X-MAP:")) {
+ const attributes = parseHlsAttributes(tagValue(line));
+ if (attributes.BYTERANGE)
+ throw new UnsupportedStreamError("HLS byte ranges are not supported");
+ if (!attributes.URI)
+ throw new UnsupportedStreamError("HLS map is missing a URI");
+ resources.push({ url: attributes.URI, lineIndex, attribute: true });
+ continue;
+ }
+ if (line.startsWith("#")) continue;
+ resources.push({ url: line, lineIndex, attribute: false });
+ }
+ if (!resources.some((resource) => !resource.attribute))
+ throw new UnsupportedStreamError("HLS playlist has no media segments");
+ return resources;
+}
+
+function segmentExtension(url: string): string {
+ try {
+ const extension = extname(new URL(url).pathname).toLowerCase();
+ return /^\.[a-z0-9]{1,5}$/.test(extension) ? extension : ".bin";
+ } catch {
+ return ".bin";
+ }
+}
+
+export interface PrefetchProgress {
+ completedSegments: number;
+ totalSegments: number;
+ bytes: number;
+}
+
+export interface PrefetchOptions extends Omit<DownloadToFileOptions, "signal"> {
+ signal: AbortSignal;
+ jobSemaphore: Semaphore;
+ globalSemaphore: Semaphore;
+ maxBytes: number;
+ onProgress?: (progress: PrefetchProgress) => void;
+}
+
+export interface PrefetchResult {
+ masterPath: string;
+ bytes: number;
+ segments: number;
+}
+
+export async function prefetchMaterializedHls(
+ masterPath: string,
+ workDir: string,
+ options: PrefetchOptions,
+): Promise<PrefetchResult> {
+ const master = await readFile(masterPath, "utf8");
+ const selection = selectHlsTracks(master) ?? {
+ video: masterPath,
+ audio: null,
+ masterLines: [],
+ };
+ const tracks = [
+ { name: "video", path: selection.video },
+ ...(selection.audio ? [{ name: "audio", path: selection.audio }] : []),
+ ];
+ const playlists = await Promise.all(
+ tracks.map(async (track) => {
+ const content = await readFile(track.path, "utf8");
+ return {
+ ...track,
+ lines: content.split(/\r?\n/),
+ resources: listMediaPlaylistResources(content),
+ };
+ }),
+ );
+ const downloads = playlists.flatMap((playlist) =>
+ playlist.resources.map((resource, index) => ({
+ playlist,
+ resource,
+ path: join(
+ workDir,
+ `${playlist.name}-${String(index).padStart(6, "0")}${segmentExtension(resource.url)}`,
+ ),
+ })),
+ );
+ const failure = new AbortController();
+ const signal = AbortSignal.any([options.signal, failure.signal]);
+ const progress: PrefetchProgress = {
+ completedSegments: 0,
+ totalSegments: downloads.length,
+ bytes: 0,
+ };
+ const results = await Promise.allSettled(
+ downloads.map((download) =>
+ options.jobSemaphore
+ .run(() =>
+ options.globalSemaphore.run(async () => {
+ signal.throwIfAborted();
+ const bytes = await downloadToFile(
+ download.resource.url,
+ download.path,
+ { ...options, signal },
+ );
+ progress.bytes += bytes;
+ progress.completedSegments++;
+ if (progress.bytes > options.maxBytes) {
+ const error = new Error("Loom imports are limited to 10 GB");
+ failure.abort(error);
+ throw error;
+ }
+ options.onProgress?.({ ...progress });
+ }),
+ )
+ .catch((error) => {
+ failure.abort(error);
+ throw error;
+ }),
+ ),
+ );
+ const rejected = results.find(
+ (result): result is PromiseRejectedResult => result.status === "rejected",
+ );
+ if (rejected) {
+ options.signal.throwIfAborted();
+ throw failure.signal.reason ?? rejected.reason;
+ }
+ for (const download of downloads) {
+ const line = download.playlist.lines[download.resource.lineIndex] ?? "";
+ download.playlist.lines[download.resource.lineIndex] = download.resource
+ .attribute
+ ? line.replace(`URI="${download.resource.url}"`, `URI="${download.path}"`)
+ : download.path;
+ }
+ const localPlaylists = new Map<string, string>();
+ for (const playlist of playlists) {
+ const localPath = join(workDir, `${playlist.name}-local.m3u8`);
+ await writeFile(localPath, playlist.lines.join("\n"));
+ localPlaylists.set(playlist.path, localPath);
+ }
+ const localMasterPath = join(workDir, "master-local.m3u8");
+ if (selection.masterLines.length > 0) {
+ await writeFile(
+ localMasterPath,
+ selection.masterLines
+ .map((line) => {
+ for (const [remote, local] of localPlaylists) {
+ if (line === remote) return local;
+ if (line.includes(`URI="${remote}"`))
+ return line.replace(`URI="${remote}"`, `URI="${local}"`);
+ }
+ return line;
+ })
+ .join("\n"),
+ );
+ } else {
+ await writeFile(
+ localMasterPath,
+ await readFile(localPlaylists.get(masterPath) ?? masterPath, "utf8"),
+ );
+ }
+ return {
+ masterPath: localMasterPath,
+ bytes: progress.bytes,
+ segments: downloads.length,
+ };
+}