Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
feat: add a dedicated Loom importer entrypoint to the media server
Richie McIlroy7cf3198ede1f
diff --git a/apps/media-server/src/__tests__/loom-importer/import.integration.test.ts b/apps/media-server/src/__tests__/loom-importer/import.integration.test.tsnew file mode 100644index 000000000..b003dde6c--- /dev/null+++ b/apps/media-server/src/__tests__/loom-importer/import.integration.test.ts@@ -0,0 +1,385 @@+import { afterAll, beforeAll, describe, expect, test } from "bun:test";+import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { spawnSync } from "bun";+import { createJob, generateJobId, getJob } from "../../lib/job-manager";+import { Semaphore } from "../../loom-importer/fetch";+import {+ type ImporterSettings,+ type ImportRequest,+ runLoomImport,+} from "../../loom-importer/job";++const SIGNATURE = "Policy=signed&Signature=s&Key-Pair-Id=k";++function ffmpeg(args: string[], cwd: string) {+ const result = spawnSync({+ cmd: ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", ...args],+ cwd,+ stderr: "pipe",+ });+ if (result.exitCode !== 0)+ throw new Error(result.stderr.toString().slice(-2000));+}++async function ffmpegAsync(args: string[], cwd: string) {+ const proc = Bun.spawn({+ cmd: ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", ...args],+ cwd,+ stderr: "pipe",+ });+ const [stderr, exitCode] = await Promise.all([+ new Response(proc.stderr).text(),+ proc.exited,+ ]);+ if (exitCode !== 0) throw new Error(stderr.slice(-2000));+}++function probe(path: string) {+ const result = spawnSync({+ cmd: [+ "ffprobe",+ "-v",+ "error",+ "-show_entries",+ "stream=codec_type,codec_name,width,start_time:format=duration",+ "-of",+ "json",+ path,+ ],+ stderr: "pipe",+ });+ if (result.exitCode !== 0) throw new Error(result.stderr.toString());+ return JSON.parse(result.stdout.toString()) as {+ streams: Array<{+ codec_type: string;+ codec_name: string;+ width?: number;+ start_time: string;+ }>;+ format: { duration: string };+ };+}++function streamStart(output: ReturnType<typeof probe>, type: string) {+ return Number(+ output.streams.find((stream) => stream.codec_type === type)?.start_time,+ );+}++describe("runLoomImport", () => {+ let root: string;+ let server: ReturnType<typeof Bun.serve>;+ let base: string;+ const uploads = new Map<string, Uint8Array>();+ const webhooks: Array<Record<string, unknown>> = [];+ const sourceRequests: string[] = [];++ beforeAll(async () => {+ root = await mkdtemp(join(tmpdir(), "loom-importer-test-"));+ const hls = join(root, "hls");+ const dash = join(root, "dash");+ await mkdir(hls);+ await mkdir(dash);+ ffmpeg(+ [+ "-f",+ "lavfi",+ "-i",+ "testsrc2=size=320x240:rate=30",+ "-itsoffset",+ "0.25",+ "-f",+ "lavfi",+ "-i",+ "sine=frequency=440:sample_rate=48000",+ "-t",+ "9",+ "-map",+ "0:v",+ "-map",+ "0:v",+ "-map",+ "1:a",+ "-c:v",+ "libx264",+ "-preset",+ "ultrafast",+ "-g",+ "30",+ "-s:v:1",+ "160x120",+ "-b:v:0",+ "600k",+ "-b:v:1",+ "150k",+ "-c:a",+ "aac",+ "-f",+ "hls",+ "-hls_time",+ "2",+ "-hls_playlist_type",+ "vod",+ "-master_pl_name",+ "master.m3u8",+ "-var_stream_map",+ "v:0,agroup:aud v:1,agroup:aud a:0,agroup:aud,default:yes",+ "-hls_segment_filename",+ "stream%v-%03d.ts",+ "stream%v.m3u8",+ ],+ hls,+ );+ ffmpeg(+ [+ "-f",+ "lavfi",+ "-i",+ "testsrc2=size=320x240:rate=30",+ "-f",+ "lavfi",+ "-i",+ "sine=frequency=440:sample_rate=48000",+ "-t",+ "9",+ "-c:v",+ "libvpx-vp9",+ "-deadline",+ "realtime",+ "-cpu-used",+ "8",+ "-b:v",+ "300k",+ "-g",+ "60",+ "-c:a",+ "libopus",+ "-f",+ "dash",+ "-seg_duration",+ "2",+ "-use_template",+ "1",+ "-use_timeline",+ "1",+ "-dash_segment_type",+ "webm",+ "manifest.mpd",+ ],+ dash,+ );+ server = Bun.serve({+ port: 0,+ hostname: "127.0.0.1",+ async fetch(request) {+ const url = new URL(request.url);+ if (url.pathname.startsWith("/open/")) {+ const file = Bun.file(join(root, url.pathname.slice(6)));+ return (await file.exists())+ ? new Response(file)+ : new Response("missing", { status: 404 });+ }+ if (url.pathname.startsWith("/media/")) {+ sourceRequests.push(url.pathname);+ if (url.searchParams.get("Signature") !== "s")+ return new Response("missing signature", { status: 403 });+ const file = Bun.file(join(root, url.pathname.slice(7)));+ return (await file.exists())+ ? new Response(file)+ : new Response("missing", { status: 404 });+ }+ if (url.pathname.startsWith("/put/") && request.method === "PUT") {+ uploads.set(+ url.pathname.slice(5),+ new Uint8Array(await request.arrayBuffer()),+ );+ return new Response(null, {+ status: 200,+ headers: { ETag: '"etag"' },+ });+ }+ if (url.pathname === "/webhook") {+ webhooks.push(await request.json());+ return Response.json({ success: true });+ }+ return new Response("not found", { status: 404 });+ },+ });+ base = `http://127.0.0.1:${server.port}`;+ }, 120_000);++ afterAll(async () => {+ server?.stop(true);+ if (root) await rm(root, { recursive: true, force: true });+ });++ const settings = (): ImporterSettings => ({+ workRoot: join(root, "work"),+ allowedHosts: ["127.0.0.1"],+ globalSemaphore: new Semaphore(64),+ jobConnections: 8,+ rangedConnections: 4,+ segmentAttempts: 2,+ segmentTimeoutMs: 10_000,+ remuxTimeoutMs: 60_000,+ terminalWebhookAttempts: 2,+ minFreeBytes: 1,+ });++ const request = (+ videoId: string,+ videoUrl: string,+ extra: Partial<ImportRequest> = {},+ ): ImportRequest => ({+ videoId,+ userId: "user",+ videoUrl,+ outputPresignedUrl: `${base}/put/${videoId}/result.mp4`,+ thumbnailPresignedUrl: `${base}/put/${videoId}/thumbnail.jpg`,+ previewGifPresignedUrl: `${base}/put/${videoId}/preview.gif`,+ webhookUrl: `${base}/webhook`,+ ...extra,+ });++ async function run(input: ImportRequest) {+ const job = createJob(+ generateJobId(),+ input.videoId,+ input.userId,+ input.webhookUrl,+ );+ await runLoomImport(+ job,+ input,+ settings(),+ new Semaphore(8),+ new AbortController().signal,+ );+ return getJob(job.jobId);+ }++ test("imports the best HLS variant as a playable MP4 with unchanged timing", async () => {+ const job = await run(+ request("hls-video", `${base}/media/hls/master.m3u8?${SIGNATURE}`),+ );+ expect(job?.phase).toBe("complete");+ expect(job?.deferredSourceKey).toBeUndefined();+ expect(job?.metadata).toMatchObject({+ width: 320,+ height: 240,+ fps: 30,+ videoCodec: "h264",+ audioCodec: "aac",+ });+ const uploaded = uploads.get("hls-video/result.mp4");+ expect(uploaded?.byteLength).toBeGreaterThan(10_000);+ expect(uploads.get("hls-video/thumbnail.jpg")?.byteLength).toBeGreaterThan(+ 0,+ );+ expect(uploads.get("hls-video/preview.gif")?.byteLength).toBeGreaterThan(0);+ const imported = join(root, "imported.mp4");+ await writeFile(imported, uploaded ?? new Uint8Array());+ const reference = join(root, "reference.mp4");+ await ffmpegAsync(+ [+ "-protocol_whitelist",+ "file,http,tcp",+ "-i",+ `${base}/open/hls/master.m3u8`,+ "-map",+ "0:v:0",+ "-map",+ "0:a:0",+ "-c",+ "copy",+ reference,+ ],+ root,+ );+ const importedProbe = probe(imported);+ const referenceProbe = probe(reference);+ expect(+ importedProbe.streams.find((s) => s.codec_type === "video")?.width,+ ).toBe(320);+ const offset = (output: ReturnType<typeof probe>) =>+ streamStart(output, "audio") - streamStart(output, "video");+ expect(offset(importedProbe)).toBeCloseTo(offset(referenceProbe), 3);+ expect(offset(importedProbe)).toBeCloseTo(0.25, 1);+ expect(Number(importedProbe.format.duration)).toBeCloseTo(+ Number(referenceProbe.format.duration),+ 0,+ );+ expect(sourceRequests.some((path) => path.includes("stream1-"))).toBe(+ false,+ );+ const complete = webhooks.findLast(+ (payload) => payload.videoId === "hls-video",+ );+ expect(complete?.phase).toBe("complete");+ expect(complete?.metadata).toMatchObject({ width: 320, fps: 30 });+ }, 60_000);++ test("stores VP9 DASH as the original WebM and reports it for deferred processing", async () => {+ const job = await run(+ request("dash-video", `${base}/media/dash/manifest.mpd?${SIGNATURE}`, {+ originalUpload: {+ key: "user/dash-video/raw-upload.webm",+ upload: { type: "put", url: `${base}/put/dash-video/raw.webm` },+ },+ }),+ );+ expect(job?.phase).toBe("complete");+ expect(job?.deferredSourceKey).toBe("user/dash-video/raw-upload.webm");+ expect(job?.metadata).toMatchObject({+ videoCodec: "vp9",+ audioCodec: "opus",+ width: 320,+ });+ expect(job?.metadata?.duration).toBeCloseTo(9, 0);+ expect(uploads.has("dash-video/result.mp4")).toBe(false);+ const original = join(root, "original.webm");+ await writeFile(+ original,+ uploads.get("dash-video/raw.webm") ?? new Uint8Array(),+ );+ const originalProbe = probe(original);+ expect(originalProbe.streams.map((s) => s.codec_name).sort()).toEqual([+ "opus",+ "vp9",+ ]);+ const complete = webhooks.findLast(+ (payload) => payload.videoId === "dash-video",+ );+ expect(complete?.deferredSourceKey).toBe("user/dash-video/raw-upload.webm");+ }, 60_000);++ test("fails VP9 sources that have nowhere to store the original", async () => {+ const job = await run(+ request("dash-no-target", `${base}/media/dash/manifest.mpd?${SIGNATURE}`),+ );+ expect(job?.phase).toBe("error");+ expect(job?.error).toContain("original upload target");+ }, 60_000);++ test("fails without retrying forever when segments are not signed", async () => {+ const job = await run(+ request("unsigned", `${base}/media/hls/master.m3u8?Policy=x`),+ );+ expect(job?.phase).toBe("error");+ expect(job?.error).toContain("403");+ }, 60_000);++ test("refuses sources outside the allowed hosts", async () => {+ const job = await run(+ request("foreign", "https://example.com/master.m3u8?Policy=x"),+ );+ expect(job?.phase).toBe("error");+ expect(job?.error).toContain("not allowed");+ });++ test("leaves no working files behind", async () => {+ expect(await readdir(join(root, "work"))).toEqual([]);+ });+});diff --git a/apps/media-server/src/__tests__/loom-importer/parsing.test.ts b/apps/media-server/src/__tests__/loom-importer/parsing.test.tsnew file mode 100644index 000000000..c91b4d938--- /dev/null+++ b/apps/media-server/src/__tests__/loom-importer/parsing.test.ts@@ -0,0 +1,229 @@+import { describe, expect, test } from "bun:test";+import {+ assertAllowedSourceUrl,+ LoomSourceRejectedError,+ parseAllowedHosts,+ Semaphore,+} from "../../loom-importer/fetch";+import {+ buildRemuxArgs,+ chooseContainer,+ resolveFps,+} from "../../loom-importer/media";+import {+ listMediaPlaylistResources,+ parseHlsAttributes,+ selectHlsTracks,+ UnsupportedStreamError,+} from "../../loom-importer/streaming";++describe("selectHlsTracks", () => {+ const master = [+ "#EXTM3U",+ "#EXT-X-VERSION:3",+ '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="low",NAME="low",DEFAULT=YES,URI="/m/audio-low.m3u8"',+ '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="alt",DEFAULT=NO,URI="/m/audio-alt.m3u8"',+ '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="main",DEFAULT=YES,URI="/m/audio-main.m3u8"',+ '#EXT-X-STREAM-INF:BANDWIDTH=1500000,RESOLUTION=1280x720,CODECS="avc1.4d001f,mp4a.40.2",AUDIO="low"',+ "/m/video-1500.m3u8",+ '#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=9000000,URI="/m/iframes.m3u8"',+ '#EXT-X-STREAM-INF:BANDWIDTH=5500000,RESOLUTION=2400x1350,CODECS="avc1.4d002a,mp4a.40.2",AUDIO="audio"',+ "",+ "/m/video-5500.m3u8",+ '#EXT-X-STREAM-INF:BANDWIDTH=5500000,RESOLUTION=1920x1080,AUDIO="audio"',+ "/m/video-5500-small.m3u8",+ ].join("\n");++ test("picks the highest bandwidth variant and its default audio rendition", () => {+ const selected = selectHlsTracks(master);+ expect(selected?.video).toBe("/m/video-5500.m3u8");+ expect(selected?.audio).toBe("/m/audio-main.m3u8");+ expect(selected?.masterLines).toEqual([+ "#EXTM3U",+ "#EXT-X-VERSION:3",+ '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="main",DEFAULT=YES,URI="/m/audio-main.m3u8"',+ '#EXT-X-STREAM-INF:BANDWIDTH=5500000,RESOLUTION=2400x1350,CODECS="avc1.4d002a,mp4a.40.2",AUDIO="audio"',+ "/m/video-5500.m3u8",+ ]);+ });++ test("keeps muxed audio when the variant has no audio group", () => {+ const selected = selectHlsTracks(+ "#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=10\nlow.m3u8\n#EXT-X-STREAM-INF:BANDWIDTH=20\nhigh.m3u8",+ );+ expect(selected?.video).toBe("high.m3u8");+ expect(selected?.audio).toBeNull();+ });++ test("treats a media playlist as having no variants", () => {+ expect(+ selectHlsTracks("#EXTM3U\n#EXTINF:4,\nhttps://cdn.loom.com/a.ts"),+ ).toBeNull();+ });++ test("parses quoted attribute values containing commas", () => {+ expect(+ parseHlsAttributes(+ 'BANDWIDTH=10,CODECS="avc1.4d002a,mp4a.40.2",AUDIO="a"',+ ),+ ).toEqual({ BANDWIDTH: "10", CODECS: "avc1.4d002a,mp4a.40.2", AUDIO: "a" });+ });+});++describe("listMediaPlaylistResources", () => {+ test("lists the init map and every segment with their line positions", () => {+ const resources = listMediaPlaylistResources(+ [+ "#EXTM3U",+ '#EXT-X-MAP:URI="https://luna.loom.com/init.webm?Policy=x"',+ "#EXTINF:4.0,",+ "https://luna.loom.com/v-0.webm?Policy=x",+ "#EXTINF:4.0,",+ "https://luna.loom.com/v-1.webm?Policy=x",+ "#EXT-X-ENDLIST",+ ].join("\n"),+ );+ expect(resources).toEqual([+ {+ url: "https://luna.loom.com/init.webm?Policy=x",+ lineIndex: 1,+ attribute: true,+ },+ {+ url: "https://luna.loom.com/v-0.webm?Policy=x",+ lineIndex: 3,+ attribute: false,+ },+ {+ url: "https://luna.loom.com/v-1.webm?Policy=x",+ lineIndex: 5,+ attribute: false,+ },+ ]);+ });++ test.each([+ '#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI="k"\n#EXTINF:4,\na.ts',+ "#EXTM3U\n#EXT-X-BYTERANGE:100@0\n#EXTINF:4,\na.ts",+ '#EXTM3U\n#EXT-X-MAP:URI="i.mp4",BYTERANGE="10@0"\n#EXTINF:4,\na.ts',+ "#EXTM3U\n#EXT-X-ENDLIST",+ ])("rejects playlists it cannot prefetch: %s", (playlist) => {+ expect(() => listMediaPlaylistResources(playlist)).toThrow(+ UnsupportedStreamError,+ );+ });++ test("accepts an explicit unencrypted key tag", () => {+ expect(+ listMediaPlaylistResources(+ "#EXTM3U\n#EXT-X-KEY:METHOD=NONE\n#EXTINF:4,\na.ts",+ ),+ ).toHaveLength(1);+ });+});++describe("assertAllowedSourceUrl", () => {+ const hosts = parseAllowedHosts(undefined);++ test.each([+ "https://luna.loom.com/id/x/playlist.m3u8?Policy=a",+ "https://cdn.loom.com/sessions/transcoded/x.mp4",+ "https://loom.com/x.ts",+ ])("allows Loom hosts over https: %s", (url) => {+ expect(assertAllowedSourceUrl(url, hosts).href).toBe(url);+ });++ test.each([+ "http://luna.loom.com/a.ts",+ "https://loom.com.attacker.example/a.ts",+ "https://attackerloom.com/a.ts",+ "http://169.254.169.254/latest/meta-data/",+ "https://user:[email protected]/a.ts",+ "file:///etc/passwd",+ "not a url",+ ])("rejects anything else: %s", (url) => {+ expect(() => assertAllowedSourceUrl(url, hosts)).toThrow(+ LoomSourceRejectedError,+ );+ });++ test("allows plain http only for an exact configured host", () => {+ const local = parseAllowedHosts("127.0.0.1");+ expect(assertAllowedSourceUrl("http://127.0.0.1:9/a.ts", local).host).toBe(+ "127.0.0.1:9",+ );+ expect(() =>+ assertAllowedSourceUrl("http://localhost/a.ts", local),+ ).toThrow(LoomSourceRejectedError);+ });+});++describe("media decisions", () => {+ test.each([+ [{ video: "h264", audio: "aac" }, "mp4"],+ [{ video: "h264", audio: null }, "mp4"],+ [{ video: "vp9", audio: "opus" }, "webm"],+ [{ video: "vp8", audio: "vorbis" }, "webm"],+ [{ video: "h264", audio: "opus" }, null],+ [{ video: "hevc", audio: "aac" }, null],+ [{ video: null, audio: "aac" }, null],+ ] as const)("chooses a container for %o", (codecs, expected) => {+ expect(chooseContainer(codecs)).toBe(expected);+ });++ test("uses the measured packet rate when the average rate is missing", () => {+ expect(+ resolveFps(+ {+ avg_frame_rate: "0/0",+ r_frame_rate: "1000/1",+ nb_read_packets: "600",+ },+ 20,+ ),+ ).toBe(30);+ });++ test("rejects implausible nominal rates", () => {+ expect(resolveFps({ r_frame_rate: "1000/1" }, 0)).toBeNull();+ expect(resolveFps({ avg_frame_rate: "30000/1001" }, 10)).toBe(29.97);+ });++ test("maps only the first video and audio stream", () => {+ const args = buildRemuxArgs([], "in.m3u8", "out.mp4", "mp4", true);+ expect(args.join(" ")).toContain("-map 0:v:0 -map 0:a:0");+ expect(args.join(" ")).toContain("-c copy -movflags +faststart -f mp4");+ expect(+ buildRemuxArgs([], "in.m3u8", "out.webm", "webm", false).join(" "),+ ).not.toContain("0:a:0");+ });+});++describe("Semaphore", () => {+ test("never runs more than its size at once", async () => {+ const semaphore = new Semaphore(3);+ let active = 0;+ let peak = 0;+ await Promise.all(+ Array.from({ length: 20 }, () =>+ semaphore.run(async () => {+ active++;+ peak = Math.max(peak, active);+ await new Promise((resolve) => setTimeout(resolve, 2));+ active--;+ }),+ ),+ );+ expect(peak).toBe(3);+ });++ test("releases its slot when work throws", async () => {+ const semaphore = new Semaphore(1);+ await expect(+ semaphore.run(async () => {+ throw new Error("boom");+ }),+ ).rejects.toThrow("boom");+ expect(await semaphore.run(async () => "ok")).toBe("ok");+ });+});diff --git a/apps/media-server/src/lib/job-manager.ts b/apps/media-server/src/lib/job-manager.tsindex 5c05238f2..40934074e 100644--- a/apps/media-server/src/lib/job-manager.ts+++ b/apps/media-server/src/lib/job-manager.ts@@ -77,6 +77,7 @@ export interface JobProgress {errorCode?: RecordingErrorCode;metadata?: VideoMetadata;outputUrl?: string;+ deferredSourceKey?: string;}export interface RecordingWorkerAcknowledgement {@@ -137,6 +138,7 @@ export interface Job {errorCode?: RecordingErrorCode;metadata?: VideoMetadata;outputUrl?: string;+ deferredSourceKey?: string;createdAt: number;updatedAt: number;inputTempFile?: TempFileHandle;@@ -405,6 +407,7 @@ export function updateJob(| "manifestSha256"| "recordingVerification"| "outputUrl"+ | "deferredSourceKey"| "inputTempFile"| "outputTempFile"| "mediaOperation"@@ -689,6 +692,7 @@ export function getJobProgress(job: Job): JobProgress {errorCode: job.errorCode,metadata: job.metadata,outputUrl: job.outputUrl,+ deferredSourceKey: job.deferredSourceKey,};}diff --git a/apps/media-server/src/loom-importer/app.ts b/apps/media-server/src/loom-importer/app.tsnew file mode 100644index 000000000..709add741--- /dev/null+++ b/apps/media-server/src/loom-importer/app.ts@@ -0,0 +1,211 @@+import { tmpdir } from "node:os";+import { join } from "node:path";+import { Hono } from "hono";+import { validateMediaServerSecret } from "../lib/auth";+import {+ createJob,+ deleteJob,+ generateJobId,+ getAllJobs,+ getJob,+ getJobProgress,+ type Job,+ sendWebhook,+ updateJob,+} from "../lib/job-manager";+import { LOOM_IMPORT_MAX_BYTES } from "../lib/media-video";+import { parseAllowedHosts, Semaphore } from "./fetch";+import {+ type ImporterSettings,+ type ImportRequest,+ importRequestSchema,+ runLoomImport,+} from "./job";++function positiveInt(value: string | undefined, fallback: number): number {+ const parsed = Number.parseInt(value ?? "", 10);+ return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;+}++export function loadImporterSettings(+ env: Record<string, string | undefined> = process.env,+) {+ const settings: ImporterSettings = {+ workRoot: env.LOOM_IMPORTER_WORK_DIR ?? join(tmpdir(), "loom-importer"),+ allowedHosts: parseAllowedHosts(env.LOOM_IMPORTER_ALLOWED_HOSTS),+ globalSemaphore: new Semaphore(+ positiveInt(env.LOOM_IMPORTER_TOTAL_CONNECTIONS, 384),+ ),+ jobConnections: positiveInt(env.LOOM_IMPORTER_JOB_CONNECTIONS, 48),+ rangedConnections: Math.min(+ 32,+ positiveInt(env.LOOM_IMPORTER_RANGED_CONNECTIONS, 16),+ ),+ segmentAttempts: positiveInt(env.LOOM_IMPORTER_SEGMENT_ATTEMPTS, 6),+ segmentTimeoutMs: positiveInt(env.LOOM_IMPORTER_SEGMENT_TIMEOUT_MS, 60_000),+ remuxTimeoutMs: positiveInt(env.LOOM_IMPORTER_REMUX_TIMEOUT_MS, 1_800_000),+ terminalWebhookAttempts: positiveInt(env.LOOM_IMPORTER_WEBHOOK_ATTEMPTS, 8),+ minFreeBytes:+ positiveInt(env.LOOM_IMPORTER_MIN_FREE_GB, 25) * 1024 * 1024 * 1024,+ };+ return {+ settings,+ maxActiveJobs: positiveInt(env.LOOM_IMPORTER_MAX_JOBS, 16),+ maxQueuedJobs: positiveInt(env.LOOM_IMPORTER_MAX_QUEUED, 400),+ };+}++const TERMINAL_JOB_TTL_MS = 24 * 60 * 60 * 1000;++function isActive(job: Job) {+ return (+ job.phase !== "complete" &&+ job.phase !== "error" &&+ job.phase !== "cancelled"+ );+}++export function createLoomImporter(+ config: ReturnType<typeof loadImporterSettings> = loadImporterSettings(),+) {+ const queue: Array<{ job: Job; request: ImportRequest }> = [];+ const running = new Map<string, AbortController>();+ let draining = false;++ const pump = () => {+ while (running.size < config.maxActiveJobs && queue.length > 0) {+ const next = queue.shift();+ if (!next) break;+ const controller = new AbortController();+ running.set(next.job.jobId, controller);+ void runLoomImport(+ next.job,+ next.request,+ config.settings,+ new Semaphore(config.settings.jobConnections),+ controller.signal,+ ).finally(() => {+ running.delete(next.job.jobId);+ setTimeout(+ () => deleteJob(next.job.jobId),+ TERMINAL_JOB_TTL_MS,+ ).unref?.();+ pump();+ });+ }+ };++ const app = new Hono().basePath("/loom-importer");++ app.get("/health", (c) =>+ c.json({+ status: draining ? "draining" : "ok",+ running: running.size,+ queued: queue.length,+ }),+ );++ app.get("/video/status", (c) => {+ if (!validateMediaServerSecret(c))+ return c.json({ error: "Unauthorized" }, 401);+ return c.json({+ loomImporter: true,+ loomImportMaxBytes: LOOM_IMPORT_MAX_BYTES,+ loomMultipartUploads: true,+ loomDeferredOriginals: true,+ draining,+ running: running.size,+ queued: queue.length,+ maxActiveJobs: config.maxActiveJobs,+ maxQueuedJobs: config.maxQueuedJobs,+ jobs: getAllJobs()+ .filter(isActive)+ .map((job) => ({+ jobId: job.jobId,+ videoId: job.videoId,+ phase: job.phase,+ progress: job.progress,+ ageSeconds: Math.round((Date.now() - job.createdAt) / 1000),+ })),+ });+ });++ app.post("/video/import", async (c) => {+ if (!validateMediaServerSecret(c))+ return c.json({ error: "Unauthorized" }, 401);+ const parsed = importRequestSchema.safeParse(+ await c.req.json().catch(() => null),+ );+ if (!parsed.success)+ return c.json(+ {+ error: "Invalid request",+ code: "INVALID_REQUEST",+ details: parsed.error.message,+ },+ 400,+ );+ if (draining || queue.length >= config.maxQueuedJobs) {+ c.header("Retry-After", "30");+ return c.json(+ {+ error: "Server is busy",+ code: "SERVER_BUSY",+ details: draining+ ? "Loom importer is draining"+ : "Loom import queue is full",+ },+ 503,+ );+ }+ const request = parsed.data;+ const job = createJob(+ generateJobId(),+ request.videoId,+ request.userId,+ request.webhookUrl,+ request.webhookSecret,+ );+ queue.push({ job, request });+ pump();+ return c.json({+ jobId: job.jobId,+ status: "queued",+ message: "Loom import queued",+ });+ });++ app.get("/video/process/:jobId/status", (c) => {+ if (!validateMediaServerSecret(c))+ return c.json({ error: "Unauthorized" }, 401);+ const job = getJob(c.req.param("jobId"));+ if (!job) return c.json({ error: "Job not found", code: "NOT_FOUND" }, 404);+ return c.json(getJobProgress(job));+ });++ const cancelJob = async (job: Job) => {+ const cancelled = updateJob(job.jobId, {+ phase: "cancelled",+ progress: 0,+ message: "Server shutting down",+ });+ if (cancelled) await sendWebhook(cancelled);+ };++ const drain = async (timeoutMs: number) => {+ draining = true;+ const deadline = Date.now() + timeoutMs;+ while ((queue.length > 0 || running.size > 0) && Date.now() < deadline)+ await new Promise((resolve) => setTimeout(resolve, 500));+ await Promise.all(queue.splice(0).map(({ job }) => cancelJob(job)));+ for (const controller of running.values()) controller.abort();+ while (running.size > 0)+ await new Promise((resolve) => setTimeout(resolve, 250));+ };++ return {+ app,+ drain,+ stats: () => ({ running: running.size, queued: queue.length }),+ };+}diff --git a/apps/media-server/src/loom-importer/fetch.ts b/apps/media-server/src/loom-importer/fetch.tsnew file mode 100644index 000000000..e01431feb--- /dev/null+++ b/apps/media-server/src/loom-importer/fetch.ts@@ -0,0 +1,188 @@+import { open, rename, rm } from "node:fs/promises";++export class LoomSourceRejectedError extends Error {}++export class Semaphore {+ private available: number;+ private readonly waiters: Array<() => void> = [];++ constructor(size: number) {+ this.available = Math.max(1, size);+ }++ async run<T>(work: () => Promise<T>): Promise<T> {+ if (this.available > 0) {+ this.available--;+ } else {+ await new Promise<void>((resolve) => this.waiters.push(resolve));+ }+ try {+ return await work();+ } finally {+ const next = this.waiters.shift();+ if (next) next();+ else this.available++;+ }+ }+}++export function parseAllowedHosts(value: string | undefined): string[] {+ const hosts = (value ?? "loom.com")+ .split(",")+ .map((host) => host.trim().toLowerCase())+ .filter(Boolean);+ return hosts.length > 0 ? hosts : ["loom.com"];+}++export function assertAllowedSourceUrl(+ value: string,+ allowedHosts: string[],+): URL {+ let url: URL;+ try {+ url = new URL(value);+ } catch {+ throw new LoomSourceRejectedError("Loom source URL is invalid");+ }+ const hostname = url.hostname.toLowerCase();+ const allowed = allowedHosts.some(+ (host) => hostname === host || hostname.endsWith(`.${host}`),+ );+ const secure =+ url.protocol === "https:" ||+ (url.protocol === "http:" && allowedHosts.includes(hostname));+ if (!allowed || !secure || url.username || url.password) {+ throw new LoomSourceRejectedError(+ `Loom source host is not allowed: ${url.protocol}//${hostname}`,+ );+ }+ return url;+}++export function createAllowedFetch(allowedHosts: string[]): typeof fetch {+ const allowedFetch = (input: RequestInfo | URL, init?: RequestInit) => {+ const url =+ input instanceof Request+ ? input.url+ : input instanceof URL+ ? input.href+ : input;+ assertAllowedSourceUrl(url, allowedHosts);+ return fetch(input, { ...init, redirect: "error" });+ };+ return Object.assign(allowedFetch, { preconnect: fetch.preconnect });+}++const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]);+const PERMANENT_STATUS_ATTEMPTS = 2;++function retryDelayMs(attempt: number, retryAfter: string | null): number {+ const seconds = Number(retryAfter);+ if (Number.isFinite(seconds) && seconds > 0)+ return Math.min(30_000, seconds * 1000);+ return Math.min(8_000, 500 * 2 ** attempt) + Math.random() * 250;+}++function sleep(ms: number, signal?: AbortSignal): Promise<void> {+ return new Promise((resolve, reject) => {+ if (signal?.aborted) return reject(signal.reason);+ const timer = setTimeout(resolve, ms);+ signal?.addEventListener(+ "abort",+ () => {+ clearTimeout(timer);+ reject(signal.reason);+ },+ { once: true },+ );+ });+}++export interface DownloadToFileOptions {+ fetchImpl: typeof fetch;+ signal: AbortSignal;+ attempts: number;+ requestTimeoutMs: number;+}++export async function downloadToFile(+ url: string,+ path: string,+ options: DownloadToFileOptions,+): Promise<number> {+ let lastError: unknown;+ let permanentFailures = 0;+ for (let attempt = 0; attempt < options.attempts; attempt++) {+ options.signal.throwIfAborted();+ const partialPath = `${path}.part`;+ try {+ const response = await options.fetchImpl(url, {+ signal: AbortSignal.any([+ options.signal,+ AbortSignal.timeout(options.requestTimeoutMs),+ ]),+ });+ if (!response.ok) {+ await response.body?.cancel().catch(() => {});+ const error = new Error(+ `Loom segment request failed with ${response.status}`,+ );+ if (!RETRYABLE_STATUS.has(response.status)) {+ permanentFailures++;+ if (permanentFailures >= PERMANENT_STATUS_ATTEMPTS) throw error;+ }+ lastError = error;+ await sleep(+ retryDelayMs(attempt, response.headers.get("retry-after")),+ options.signal,+ );+ continue;+ }+ const expected = Number(response.headers.get("content-length"));+ const bytes = await writeBody(response, partialPath);+ if (Number.isFinite(expected) && expected > 0 && bytes !== expected) {+ throw new Error(+ `Loom segment was truncated (${bytes} of ${expected} bytes)`,+ );+ }+ await rename(partialPath, path);+ return bytes;+ } catch (error) {+ await rm(partialPath, { force: true }).catch(() => {});+ if (options.signal.aborted) throw options.signal.reason ?? error;+ if (+ error instanceof LoomSourceRejectedError ||+ permanentFailures >= PERMANENT_STATUS_ATTEMPTS+ )+ throw error;+ lastError = error;+ await sleep(retryDelayMs(attempt, null), options.signal);+ }+ }+ throw lastError instanceof Error+ ? lastError+ : new Error("Loom segment download failed");+}++async function writeBody(response: Response, path: string): Promise<number> {+ const handle = await open(path, "w");+ let bytes = 0;+ try {+ if (!response.body) return 0;+ for await (const chunk of response.body) {+ let written = 0;+ while (written < chunk.byteLength) {+ const result = await handle.write(+ chunk,+ written,+ chunk.byteLength - written,+ );+ written += result.bytesWritten;+ }+ bytes += chunk.byteLength;+ }+ return bytes;+ } finally {+ await handle.close();+ }+}diff --git a/apps/media-server/src/loom-importer/index.ts b/apps/media-server/src/loom-importer/index.tsnew file mode 100644index 000000000..cb2d9faaf--- /dev/null+++ b/apps/media-server/src/loom-importer/index.ts@@ -0,0 +1,30 @@+import { logger } from "hono/logger";+import { getIdleTimeoutSeconds } from "../lib/lifecycle";+import { createLoomImporter } from "./app";++const port = Number(process.env.PORT) || 3456;+const drainTimeoutMs =+ Number(process.env.LOOM_IMPORTER_DRAIN_TIMEOUT_MS) || 10 * 60 * 1000;+const importer = createLoomImporter();++importer.app.use("*", logger());++let stopping = false;+const shutdown = async (signal: string) => {+ if (stopping) process.exit(1);+ stopping = true;+ console.log(`[loom-importer] ${signal}: draining`, importer.stats());+ await importer.drain(drainTimeoutMs);+ process.exit(0);+};++for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const)+ process.on(signal, () => void shutdown(signal));++console.log(`[loom-importer] Starting on port ${port}`);++export default {+ port,+ idleTimeout: getIdleTimeoutSeconds(),+ fetch: importer.app.fetch,+};diff --git a/apps/media-server/src/loom-importer/job.ts b/apps/media-server/src/loom-importer/job.tsnew file mode 100644index 000000000..ad6a1fedc--- /dev/null+++ b/apps/media-server/src/loom-importer/job.ts@@ -0,0 +1,435 @@+import { mkdir, mkdtemp, rm, statfs } from "node:fs/promises";+import { join } from "node:path";+import { z } from "zod";+import {+ type Job,+ sendWebhook,+ updateJob,+ type VideoMetadata,+} from "../lib/job-manager";+import {+ abortStorageUploadTarget,+ generatePreviewGif,+ generateThumbnail,+ getFfmpegHlsCapabilities,+ LOOM_IMPORT_MAX_BYTES,+ materializeStreamingInput,+ type StorageUploadTarget,+ uploadFileToS3,+ uploadFileToStorage,+ uploadToS3,+} from "../lib/media-video";+import { downloadUrlToPathParallel } from "../lib/parallel-download";+import type { TempFileHandle } from "../lib/temp-files";+import {+ assertAllowedSourceUrl,+ createAllowedFetch,+ type Semaphore,+} from "./fetch";+import {+ buildRemuxArgs,+ chooseContainer,+ PLAYLIST_INPUT_ARGS,+ probeCodecs,+ probeOutput,+ runCommand,+} from "./media";+import { prefetchMaterializedHls, UnsupportedStreamError } from "./streaming";++const storageUploadSchema = z.discriminatedUnion("type", [+ z.object({+ type: z.literal("put"),+ url: z.string().url(),+ ifNoneMatch: z.literal("*").optional(),+ }),+ z.object({+ type: z.literal("multipart"),+ videoId: z.string(),+ key: z.string().min(1),+ uploadId: z.string().min(1),+ partSize: z+ .number()+ .int()+ .min(5 * 1024 * 1024),+ signPartUrl: z.string().url(),+ completeUrl: z.string().url(),+ abortUrl: z.string().url(),+ webhookSecret: z.string().optional(),+ loomImportAttemptId: z.string().min(1).max(200).optional(),+ }),+]);++export const importRequestSchema = z.object({+ videoId: z.string().min(1),+ userId: z.string().min(1),+ videoUrl: z.string().url(),+ outputPresignedUrl: z.string().url(),+ outputUpload: storageUploadSchema.optional(),+ sourceUpload: storageUploadSchema.optional(),+ originalUpload: z+ .object({+ key: z.string().min(1).endsWith(".webm"),+ upload: storageUploadSchema,+ })+ .optional(),+ existingOutput: z+ .object({+ url: z.string().url(),+ objectIdentity: z.string().regex(/^"[^"\r\n]+"$/),+ fileSize: z.number().int().positive().max(LOOM_IMPORT_MAX_BYTES),+ })+ .optional(),+ maxImportBytes: z+ .number()+ .int()+ .positive()+ .max(LOOM_IMPORT_MAX_BYTES)+ .optional(),+ thumbnailPresignedUrl: z.string().url().optional(),+ previewGifPresignedUrl: z.string().url().optional(),+ webhookUrl: z.string().url().optional(),+ webhookSecret: z.string().optional(),+});++export type ImportRequest = z.infer<typeof importRequestSchema>;++export interface ImporterSettings {+ workRoot: string;+ allowedHosts: string[];+ globalSemaphore: Semaphore;+ jobConnections: number;+ rangedConnections: number;+ segmentAttempts: number;+ segmentTimeoutMs: number;+ remuxTimeoutMs: number;+ terminalWebhookAttempts: number;+ minFreeBytes: number;+}++function isStreamingUrl(url: string): boolean {+ const path = (url.split("?")[0] ?? "").toLowerCase();+ return path.endsWith(".m3u8") || path.endsWith(".mpd");+}++function withSizeLimit(fetchImpl: typeof fetch, maxBytes: number) {+ const limited = async (input: RequestInfo | URL, init?: RequestInit) => {+ const response = await fetchImpl(input, init);+ const total = Number(+ response.headers.get("content-range")?.split("/")[1] ??+ response.headers.get("content-length"),+ );+ if (Number.isFinite(total) && total > maxBytes) {+ await response.body?.cancel().catch(() => {});+ throw new Error("Loom imports are limited to 10 GB");+ }+ return response;+ };+ return Object.assign(limited, { preconnect: fetch.preconnect });+}++async function deliverTerminalWebhook(job: Job, attempts: number) {+ for (let attempt = 0; attempt < attempts; attempt++) {+ await sendWebhook(job);+ if (job.terminalWebhookAcknowledgedAt || !job.webhookUrl) return;+ await new Promise((resolve) =>+ setTimeout(resolve, Math.min(60_000, 2_000 * 2 ** attempt)),+ );+ }+ console.error(+ `[loom-importer] Terminal webhook was not acknowledged for ${job.jobId}`,+ );+}++function progressUpdate(+ jobId: string,+ phase: Job["phase"],+ progress: number,+ message: string,+) {+ const job = updateJob(jobId, { phase, progress, message });+ if (job) void sendWebhook(job);+}++interface PreparedSource {+ input: string;+ inputArgs: string[];+}++async function prepareSource(+ job: Job,+ request: ImportRequest,+ workDir: string,+ settings: ImporterSettings,+ jobSemaphore: Semaphore,+ signal: AbortSignal,+): Promise<PreparedSource> {+ const maxBytes = request.maxImportBytes ?? LOOM_IMPORT_MAX_BYTES;+ const allowedFetch = createAllowedFetch(settings.allowedHosts);+ assertAllowedSourceUrl(request.videoUrl, settings.allowedHosts);+ if (!isStreamingUrl(request.videoUrl)) {+ const sourcePath = join(workDir, "source.mp4");+ const result = await downloadUrlToPathParallel(+ request.videoUrl,+ sourcePath,+ {+ concurrency: settings.rangedConnections,+ partBytes: 8 * 1024 * 1024,+ signal,+ fetchImpl: withSizeLimit(allowedFetch, maxBytes),+ },+ );+ if (result.bytes > maxBytes)+ throw new Error("Loom imports are limited to 10 GB");+ return { input: sourcePath, inputArgs: [] };+ }+ const manifestDir = join(workDir, "manifest");+ const segmentDir = join(workDir, "segments");+ await mkdir(manifestDir);+ await mkdir(segmentDir);+ const master = await materializeStreamingInput(+ request.videoUrl,+ manifestDir,+ signal,+ );+ if (!master.endsWith(".m3u8"))+ throw new UnsupportedStreamError("This Loom DASH layout is not supported");+ const prefetched = await prefetchMaterializedHls(master, segmentDir, {+ fetchImpl: allowedFetch,+ signal,+ attempts: settings.segmentAttempts,+ requestTimeoutMs: settings.segmentTimeoutMs,+ jobSemaphore,+ globalSemaphore: settings.globalSemaphore,+ maxBytes,+ onProgress: (progress) => {+ if (+ progress.completedSegments % 50 === 0 ||+ progress.completedSegments === progress.totalSegments+ )+ progressUpdate(+ job.jobId,+ "downloading",+ Math.round(+ (progress.completedSegments / progress.totalSegments) * 60,+ ),+ `Downloading from Loom (${progress.completedSegments}/${progress.totalSegments})...`,+ );+ },+ });+ return {+ input: prefetched.masterPath,+ inputArgs: PLAYLIST_INPUT_ARGS(await getFfmpegHlsCapabilities()),+ };+}++async function uploadPreviews(+ outputPath: string,+ metadata: VideoMetadata,+ request: ImportRequest,+ signal: AbortSignal,+) {+ const thumbnail = request.thumbnailPresignedUrl+ ? generateThumbnail(outputPath, metadata.duration, {}, signal)+ : undefined;+ const gif: Promise<TempFileHandle> | undefined =+ request.previewGifPresignedUrl+ ? generatePreviewGif(outputPath, metadata.duration, {}, signal)+ : undefined;+ thumbnail?.catch(() => {});+ gif?.catch(() => {});+ try {+ if (thumbnail && request.thumbnailPresignedUrl) {+ const data = await thumbnail;+ await uploadToS3(+ data,+ request.thumbnailPresignedUrl,+ "image/jpeg",+ signal,+ );+ }+ } catch (error) {+ signal.throwIfAborted();+ console.warn("[loom-importer] Thumbnail failed", error);+ }+ try {+ if (gif && request.previewGifPresignedUrl) {+ const rendered = await gif;+ try {+ await uploadFileToS3(+ rendered.path,+ request.previewGifPresignedUrl,+ "image/gif",+ signal,+ );+ } finally {+ await rendered.cleanup();+ }+ }+ } catch (error) {+ signal.throwIfAborted();+ console.warn("[loom-importer] Preview GIF failed", error);+ }+}++async function waitForDiskSpace(+ path: string,+ minFreeBytes: number,+ signal: AbortSignal,+) {+ while (true) {+ signal.throwIfAborted();+ const stats = await statfs(path).catch(() => null);+ if (!stats || stats.bavail * stats.bsize >= minFreeBytes) return;+ await new Promise((resolve) => setTimeout(resolve, 5_000));+ }+}++export async function runLoomImport(+ job: Job,+ request: ImportRequest,+ settings: ImporterSettings,+ jobSemaphore: Semaphore,+ signal: AbortSignal,+): Promise<void> {+ const used = new Set<StorageUploadTarget>();+ const targets = [+ request.outputUpload,+ request.sourceUpload,+ request.originalUpload?.upload,+ ].filter((target): target is StorageUploadTarget => target !== undefined);+ let workDir: string | undefined;+ try {+ await mkdir(settings.workRoot, { recursive: true });+ await waitForDiskSpace(settings.workRoot, settings.minFreeBytes, signal);+ workDir = await mkdtemp(join(settings.workRoot, `${request.videoId}-`));+ progressUpdate(job.jobId, "downloading", 0, "Downloading from Loom...");+ let outputPath: string;+ let deferredSourceKey: string | undefined;+ if (request.existingOutput) {+ outputPath = join(workDir, "result.mp4");+ await downloadUrlToPathParallel(request.existingOutput.url, outputPath, {+ concurrency: settings.rangedConnections,+ partBytes: 8 * 1024 * 1024,+ signal,+ });+ } else {+ const source = await prepareSource(+ job,+ request,+ workDir,+ settings,+ jobSemaphore,+ signal,+ );+ const codecs = await probeCodecs(source.inputArgs, source.input, signal);+ const container = chooseContainer(codecs);+ if (!container)+ throw new UnsupportedStreamError(+ `Loom source codecs are not supported (${codecs.video ?? "none"}/${codecs.audio ?? "none"})`,+ );+ if (container === "webm" && !request.originalUpload)+ throw new UnsupportedStreamError(+ "This Loom recording needs an original upload target",+ );+ progressUpdate(job.jobId, "processing", 65, "Packaging video...");+ outputPath = join(+ workDir,+ container === "mp4" ? "result.mp4" : "original.webm",+ );+ await runCommand(+ buildRemuxArgs(+ source.inputArgs,+ source.input,+ outputPath,+ container,+ codecs.audio !== null,+ ),+ settings.remuxTimeoutMs,+ signal,+ );+ if (container === "webm") deferredSourceKey = request.originalUpload?.key;+ }+ const metadata = await probeOutput(outputPath, signal);+ if (metadata.fileSize > (request.maxImportBytes ?? LOOM_IMPORT_MAX_BYTES))+ throw new Error("Loom imports are limited to 10 GB");+ updateJob(job.jobId, { metadata });+ progressUpdate(job.jobId, "uploading", 75, "Saving video...");+ if (!request.existingOutput) {+ if (deferredSourceKey && request.originalUpload) {+ used.add(request.originalUpload.upload);+ await uploadFileToStorage(+ outputPath,+ request.originalUpload.upload,+ "video/webm",+ signal,+ );+ } else if (request.outputUpload) {+ used.add(request.outputUpload);+ await uploadFileToStorage(+ outputPath,+ request.outputUpload,+ "video/mp4",+ signal,+ );+ } else {+ await uploadFileToS3(+ outputPath,+ request.outputPresignedUrl,+ "video/mp4",+ signal,+ );+ }+ }+ await Promise.all(+ targets+ .filter((target) => !used.has(target))+ .map((target) => {+ used.add(target);+ return abortStorageUploadTarget(target).catch((error) =>+ console.warn("[loom-importer] Upload cleanup failed", error),+ );+ }),+ );+ progressUpdate(+ job.jobId,+ "generating_thumbnail",+ 90,+ "Generating preview assets...",+ );+ await uploadPreviews(outputPath, metadata, request, signal);+ signal.throwIfAborted();+ const completed = updateJob(job.jobId, {+ phase: "complete",+ progress: 100,+ message: deferredSourceKey+ ? "Imported original Loom video"+ : "Loom video imported",+ deferredSourceKey,+ });+ if (completed)+ await deliverTerminalWebhook(completed, settings.terminalWebhookAttempts);+ } catch (error) {+ await Promise.all(+ targets+ .filter((target) => !used.has(target))+ .map((target) => abortStorageUploadTarget(target).catch(() => {})),+ );+ const cancelled = signal.aborted;+ if (!cancelled) console.error(`[loom-importer] ${job.jobId} failed`, error);+ const failed = updateJob(job.jobId, {+ phase: cancelled ? "cancelled" : "error",+ progress: 0,+ error: cancelled+ ? undefined+ : error instanceof Error+ ? error.message+ : String(error),+ message: cancelled ? "Server shutting down" : "Loom import failed",+ });+ if (failed)+ await deliverTerminalWebhook(failed, settings.terminalWebhookAttempts);+ } finally {+ if (workDir)+ await rm(workDir, { recursive: true, force: true }).catch(() => {});+ }+}diff --git a/apps/media-server/src/loom-importer/media.ts b/apps/media-server/src/loom-importer/media.tsnew file mode 100644index 000000000..93f822718--- /dev/null+++ b/apps/media-server/src/loom-importer/media.ts@@ -0,0 +1,233 @@+import { stat } from "node:fs/promises";+import { spawn } from "bun";+import type { VideoMetadata } from "../lib/job-manager";+import type { FfmpegHlsCapabilities } from "../lib/media-video";+import {+ registerSubprocess,+ terminateProcess,+ unregisterSubprocess,+} from "../lib/subprocess";++const STDERR_LIMIT_BYTES = 64 * 1024;++export const PLAYLIST_INPUT_ARGS = (capabilities: FfmpegHlsCapabilities) => [+ "-protocol_whitelist",+ "file,crypto,data",+ "-allowed_extensions",+ "ALL",+ ...(capabilities.allowedSegmentExtensions+ ? ["-allowed_segment_extensions", "ALL"]+ : []),+ ...(capabilities.extensionPicky ? ["-extension_picky", "0"] : []),+];++export async function runCommand(+ cmd: string[],+ timeoutMs: number,+ signal: AbortSignal,+): Promise<string> {+ signal.throwIfAborted();+ const proc = registerSubprocess(+ spawn({ cmd, stdin: "ignore", stdout: "pipe", stderr: "pipe" }),+ );+ const stop = () => {+ void terminateProcess(proc);+ };+ signal.addEventListener("abort", stop, { once: true });+ const timer = setTimeout(stop, timeoutMs);+ try {+ const [stdout, stderr, exitCode] = await Promise.all([+ new Response(proc.stdout).text(),+ new Response(proc.stderr).text(),+ proc.exited,+ ]);+ signal.throwIfAborted();+ if (exitCode !== 0) {+ throw new Error(+ `${cmd[0]} exited with code ${exitCode}: ${stderr.slice(-STDERR_LIMIT_BYTES).trim().slice(-600)}`,+ );+ }+ return stdout;+ } finally {+ clearTimeout(timer);+ signal.removeEventListener("abort", stop);+ unregisterSubprocess(proc);+ }+}++interface ProbeStream {+ codec_type?: string;+ codec_name?: string;+ width?: number;+ height?: number;+ avg_frame_rate?: string;+ r_frame_rate?: string;+ nb_read_packets?: string;+ channels?: number;+ sample_rate?: string;+}++interface ProbeOutput {+ streams?: ProbeStream[];+ format?: { duration?: string; bit_rate?: string };+}++export interface SourceCodecs {+ video: string | null;+ audio: string | null;+}++export async function probeCodecs(+ inputArgs: string[],+ input: string,+ signal: AbortSignal,+): Promise<SourceCodecs> {+ const output = JSON.parse(+ await runCommand(+ [+ "ffprobe",+ "-v",+ "error",+ ...inputArgs,+ "-show_entries",+ "stream=codec_type,codec_name",+ "-of",+ "json",+ input,+ ],+ 120_000,+ signal,+ ),+ ) as ProbeOutput;+ const streams = output.streams ?? [];+ return {+ video:+ streams.find((stream) => stream.codec_type === "video")?.codec_name ??+ null,+ audio:+ streams.find((stream) => stream.codec_type === "audio")?.codec_name ??+ null,+ };+}++function parseRate(value: string | undefined): number {+ if (!value) return 0;+ const [numerator, denominator] = value.split("/").map(Number);+ if (!numerator || !denominator) return 0;+ return numerator / denominator;+}++export function resolveFps(+ stream: ProbeStream,+ duration: number,+): number | null {+ const average = parseRate(stream.avg_frame_rate);+ if (average > 0 && average <= 240) return Math.round(average * 100) / 100;+ const packets = Number(stream.nb_read_packets);+ if (packets > 0 && duration > 0) {+ const measured = packets / duration;+ if (measured > 0 && measured <= 240)+ return Math.round(measured * 100) / 100;+ }+ const nominal = parseRate(stream.r_frame_rate);+ if (nominal > 0 && nominal <= 240) return Math.round(nominal * 100) / 100;+ return null;+}++export async function probeOutput(+ path: string,+ signal: AbortSignal,+): Promise<VideoMetadata> {+ const output = JSON.parse(+ await runCommand(+ [+ "ffprobe",+ "-v",+ "error",+ "-count_packets",+ "-show_entries",+ "format=duration,bit_rate:stream=codec_type,codec_name,width,height,avg_frame_rate,r_frame_rate,nb_read_packets,channels,sample_rate",+ "-of",+ "json",+ path,+ ],+ 10 * 60_000,+ signal,+ ),+ ) as ProbeOutput;+ const streams = output.streams ?? [];+ const video = streams.find((stream) => stream.codec_type === "video");+ const audio = streams.find((stream) => stream.codec_type === "audio");+ const duration = Number(output.format?.duration);+ if (!video?.codec_name || !video.width || !video.height)+ throw new Error("Imported Loom video has no readable video stream");+ if (!Number.isFinite(duration) || duration <= 0)+ throw new Error("Imported Loom video has no readable duration");+ const fps = resolveFps(video, duration);+ if (!fps) throw new Error("Imported Loom video has no readable frame rate");+ const fileSize = (await stat(path)).size;+ return {+ duration,+ width: video.width,+ height: video.height,+ fps,+ videoCodec: video.codec_name,+ audioCodec: audio?.codec_name ?? null,+ audioChannels: audio?.channels ?? null,+ sampleRate: audio?.sample_rate ? Number(audio.sample_rate) : null,+ bitrate:+ Number(output.format?.bit_rate) || Math.round((fileSize * 8) / duration),+ fileSize,+ };+}++export type ImportContainer = "mp4" | "webm";++export function chooseContainer(codecs: SourceCodecs): ImportContainer | null {+ if (+ codecs.video === "h264" &&+ (codecs.audio === null || codecs.audio === "aac")+ )+ return "mp4";+ if (+ (codecs.video === "vp9" ||+ codecs.video === "vp8" ||+ codecs.video === "av1") &&+ (codecs.audio === null ||+ codecs.audio === "opus" ||+ codecs.audio === "vorbis")+ )+ return "webm";+ return null;+}++export function buildRemuxArgs(+ inputArgs: string[],+ input: string,+ output: string,+ container: ImportContainer,+ hasAudio: boolean,+): string[] {+ return [+ "ffmpeg",+ "-hide_banner",+ "-nostdin",+ "-loglevel",+ "error",+ ...inputArgs,+ "-i",+ input,+ "-map",+ "0:v:0",+ ...(hasAudio ? ["-map", "0:a:0"] : []),+ "-dn",+ "-sn",+ "-c",+ "copy",+ ...(container === "mp4"+ ? ["-movflags", "+faststart", "-f", "mp4"]+ : ["-f", "webm"]),+ "-y",+ output,+ ];+}diff --git a/apps/media-server/src/loom-importer/streaming.ts b/apps/media-server/src/loom-importer/streaming.tsnew file mode 100644index 000000000..06ddd19c8--- /dev/null+++ b/apps/media-server/src/loom-importer/streaming.ts@@ -0,0 +1,304 @@+import { readFile, writeFile } from "node:fs/promises";+import { extname, join } from "node:path";+import {+ type DownloadToFileOptions,+ downloadToFile,+ type Semaphore,+} from "./fetch";++export class UnsupportedStreamError extends Error {}++export interface HlsVariant {+ bandwidth: number;+ pixels: number;+ audioGroup: string | null;+ uri: string;+ line: string;+}++export interface HlsAudioRendition {+ groupId: string;+ isDefault: boolean;+ uri: string | null;+ line: string;+}++export interface SelectedTracks {+ video: string;+ audio: string | null;+ masterLines: string[];+}++export function parseHlsAttributes(source: string): Record<string, string> {+ const attributes: Record<string, string> = {};+ const pattern = /([A-Z0-9-]+)=("[^"]*"|[^,]*)/g;+ for (const match of source.matchAll(pattern)) {+ const [, key, raw] = match;+ if (!key || raw === undefined) continue;+ attributes[key] =+ raw.startsWith('"') && raw.endsWith('"') ? raw.slice(1, -1) : raw;+ }+ return attributes;+}++function tagValue(line: string): string {+ return line.slice(line.indexOf(":") + 1);+}++export function selectHlsTracks(master: string): SelectedTracks | null {+ const lines = master.split(/\r?\n/);+ const variants: HlsVariant[] = [];+ const audio: HlsAudioRendition[] = [];+ for (let index = 0; index < lines.length; index++) {+ const line = lines[index]?.trim() ?? "";+ if (line.startsWith("#EXT-X-STREAM-INF:")) {+ const attributes = parseHlsAttributes(tagValue(line));+ let uriIndex = index + 1;+ while (+ uriIndex < lines.length &&+ (!lines[uriIndex]?.trim() || lines[uriIndex]?.trim().startsWith("#"))+ )+ uriIndex++;+ const uri = lines[uriIndex]?.trim();+ if (!uri) continue;+ const [width, height] = (attributes.RESOLUTION ?? "0x0")+ .split("x")+ .map(Number);+ variants.push({+ bandwidth: Number(attributes.BANDWIDTH) || 0,+ pixels: (width || 0) * (height || 0),+ audioGroup: attributes.AUDIO ?? null,+ uri,+ line,+ });+ index = uriIndex;+ } else if (line.startsWith("#EXT-X-MEDIA:")) {+ const attributes = parseHlsAttributes(tagValue(line));+ if (attributes.TYPE !== "AUDIO" || !attributes["GROUP-ID"]) continue;+ audio.push({+ groupId: attributes["GROUP-ID"],+ isDefault: attributes.DEFAULT === "YES",+ uri: attributes.URI ?? null,+ line,+ });+ }+ }+ if (variants.length === 0) return null;+ const best = variants.reduce((current, candidate) =>+ candidate.bandwidth > current.bandwidth ||+ (candidate.bandwidth === current.bandwidth &&+ candidate.pixels > current.pixels)+ ? candidate+ : current,+ );+ const group = best.audioGroup+ ? audio.filter((rendition) => rendition.groupId === best.audioGroup)+ : [];+ const rendition =+ group.find((candidate) => candidate.isDefault && candidate.uri) ??+ group.find((candidate) => candidate.uri) ??+ null;+ const header = lines+ .map((line) => line.trim())+ .filter(+ (line) =>+ line === "#EXTM3U" ||+ line.startsWith("#EXT-X-VERSION") ||+ line.startsWith("#EXT-X-INDEPENDENT-SEGMENTS"),+ );+ return {+ video: best.uri,+ audio: rendition?.uri ?? null,+ masterLines: [+ ...(header.length > 0 ? header : ["#EXTM3U"]),+ ...(rendition ? [rendition.line] : []),+ best.line,+ best.uri,+ ],+ };+}++export interface MediaPlaylistResource {+ url: string;+ lineIndex: number;+ attribute: boolean;+}++export function listMediaPlaylistResources(+ playlist: string,+): MediaPlaylistResource[] {+ const lines = playlist.split(/\r?\n/);+ const resources: MediaPlaylistResource[] = [];+ for (const [lineIndex, raw] of lines.entries()) {+ const line = raw.trim();+ if (!line) continue;+ if (line.startsWith("#EXT-X-KEY:")) {+ const method = parseHlsAttributes(tagValue(line)).METHOD;+ if (method && method !== "NONE")+ throw new UnsupportedStreamError("Encrypted HLS is not supported");+ continue;+ }+ if (line.startsWith("#EXT-X-BYTERANGE"))+ throw new UnsupportedStreamError("HLS byte ranges are not supported");+ if (line.startsWith("#EXT-X-MAP:")) {+ const attributes = parseHlsAttributes(tagValue(line));+ if (attributes.BYTERANGE)+ throw new UnsupportedStreamError("HLS byte ranges are not supported");+ if (!attributes.URI)+ throw new UnsupportedStreamError("HLS map is missing a URI");+ resources.push({ url: attributes.URI, lineIndex, attribute: true });+ continue;+ }+ if (line.startsWith("#")) continue;+ resources.push({ url: line, lineIndex, attribute: false });+ }+ if (!resources.some((resource) => !resource.attribute))+ throw new UnsupportedStreamError("HLS playlist has no media segments");+ return resources;+}++function segmentExtension(url: string): string {+ try {+ const extension = extname(new URL(url).pathname).toLowerCase();+ return /^\.[a-z0-9]{1,5}$/.test(extension) ? extension : ".bin";+ } catch {+ return ".bin";+ }+}++export interface PrefetchProgress {+ completedSegments: number;+ totalSegments: number;+ bytes: number;+}++export interface PrefetchOptions extends Omit<DownloadToFileOptions, "signal"> {+ signal: AbortSignal;+ jobSemaphore: Semaphore;+ globalSemaphore: Semaphore;+ maxBytes: number;+ onProgress?: (progress: PrefetchProgress) => void;+}++export interface PrefetchResult {+ masterPath: string;+ bytes: number;+ segments: number;+}++export async function prefetchMaterializedHls(+ masterPath: string,+ workDir: string,+ options: PrefetchOptions,+): Promise<PrefetchResult> {+ const master = await readFile(masterPath, "utf8");+ const selection = selectHlsTracks(master) ?? {+ video: masterPath,+ audio: null,+ masterLines: [],+ };+ const tracks = [+ { name: "video", path: selection.video },+ ...(selection.audio ? [{ name: "audio", path: selection.audio }] : []),+ ];+ const playlists = await Promise.all(+ tracks.map(async (track) => {+ const content = await readFile(track.path, "utf8");+ return {+ ...track,+ lines: content.split(/\r?\n/),+ resources: listMediaPlaylistResources(content),+ };+ }),+ );+ const downloads = playlists.flatMap((playlist) =>+ playlist.resources.map((resource, index) => ({+ playlist,+ resource,+ path: join(+ workDir,+ `${playlist.name}-${String(index).padStart(6, "0")}${segmentExtension(resource.url)}`,+ ),+ })),+ );+ const failure = new AbortController();+ const signal = AbortSignal.any([options.signal, failure.signal]);+ const progress: PrefetchProgress = {+ completedSegments: 0,+ totalSegments: downloads.length,+ bytes: 0,+ };+ const results = await Promise.allSettled(+ downloads.map((download) =>+ options.jobSemaphore+ .run(() =>+ options.globalSemaphore.run(async () => {+ signal.throwIfAborted();+ const bytes = await downloadToFile(+ download.resource.url,+ download.path,+ { ...options, signal },+ );+ progress.bytes += bytes;+ progress.completedSegments++;+ if (progress.bytes > options.maxBytes) {+ const error = new Error("Loom imports are limited to 10 GB");+ failure.abort(error);+ throw error;+ }+ options.onProgress?.({ ...progress });+ }),+ )+ .catch((error) => {+ failure.abort(error);+ throw error;+ }),+ ),+ );+ const rejected = results.find(+ (result): result is PromiseRejectedResult => result.status === "rejected",+ );+ if (rejected) {+ options.signal.throwIfAborted();+ throw failure.signal.reason ?? rejected.reason;+ }+ for (const download of downloads) {+ const line = download.playlist.lines[download.resource.lineIndex] ?? "";+ download.playlist.lines[download.resource.lineIndex] = download.resource+ .attribute+ ? line.replace(`URI="${download.resource.url}"`, `URI="${download.path}"`)+ : download.path;+ }+ const localPlaylists = new Map<string, string>();+ for (const playlist of playlists) {+ const localPath = join(workDir, `${playlist.name}-local.m3u8`);+ await writeFile(localPath, playlist.lines.join("\n"));+ localPlaylists.set(playlist.path, localPath);+ }+ const localMasterPath = join(workDir, "master-local.m3u8");+ if (selection.masterLines.length > 0) {+ await writeFile(+ localMasterPath,+ selection.masterLines+ .map((line) => {+ for (const [remote, local] of localPlaylists) {+ if (line === remote) return local;+ if (line.includes(`URI="${remote}"`))+ return line.replace(`URI="${remote}"`, `URI="${local}"`);+ }+ return line;+ })+ .join("\n"),+ );+ } else {+ await writeFile(+ localMasterPath,+ await readFile(localPlaylists.get(masterPath) ?? masterPath, "utf8"),+ );+ }+ return {+ masterPath: localMasterPath,+ bytes: progress.bytes,+ segments: downloads.length,+ };+}