Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(loom-importer): import audio-only Looms and skip a missing Loom segment

Richie McIlroy0e2c0d3faf74
diff --git a/apps/media-server/src/__tests__/loom-importer/import.integration.test.ts b/apps/media-server/src/__tests__/loom-importer/import.integration.test.ts
index b003dde6c..90938ad0c 100644
--- a/apps/media-server/src/__tests__/loom-importer/import.integration.test.ts
+++ b/apps/media-server/src/__tests__/loom-importer/import.integration.test.ts
@@ -1,5 +1,13 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
-import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
+import {
+ cp,
+ mkdir,
+ mkdtemp,
+ readdir,
+ readFile,
+ rm,
+ writeFile,
+} from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { spawnSync } from "bun";
@@ -170,6 +178,75 @@ describe("runLoomImport", () => {
],
dash,
);
+ const manifest = await readFile(join(dash, "manifest.mpd"), "utf8");
+ await writeFile(
+ join(dash, "manifest-zero.mpd"),
+ manifest.replaceAll(
+ "</SegmentTimeline>",
+ '<S d="0" /></SegmentTimeline>',
+ ),
+ );
+ await cp(hls, join(root, "hls-gap"), { recursive: true });
+ await rm(join(root, "hls-gap", "stream2-002.ts"));
+ await cp(hls, join(root, "hls-holes"), { recursive: true });
+ await rm(join(root, "hls-holes", "stream2-001.ts"));
+ await rm(join(root, "hls-holes", "stream2-003.ts"));
+ const audioOnly = join(root, "hls-audio");
+ await mkdir(audioOnly);
+ ffmpeg(
+ [
+ "-f",
+ "lavfi",
+ "-i",
+ "sine=frequency=440:sample_rate=48000",
+ "-t",
+ "7",
+ "-c:a",
+ "aac",
+ "-f",
+ "hls",
+ "-hls_time",
+ "2",
+ "-hls_playlist_type",
+ "vod",
+ "-hls_segment_filename",
+ "audio-%03d.ts",
+ "audio.m3u8",
+ ],
+ audioOnly,
+ );
+ const audioPlaylist = await readFile(join(audioOnly, "audio.m3u8"), "utf8");
+ await writeFile(
+ join(audioOnly, "video.m3u8"),
+ audioPlaylist.replaceAll("audio-", "video-"),
+ );
+ for (const entry of await readdir(audioOnly))
+ if (entry.startsWith("audio-"))
+ await writeFile(join(audioOnly, entry.replace("audio-", "video-")), "");
+ await writeFile(
+ join(audioOnly, "master.m3u8"),
+ [
+ "#EXTM3U",
+ "#EXT-X-VERSION:3",
+ '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="audio",DEFAULT="YES",URI="audio.m3u8"',
+ '#EXT-X-STREAM-INF:BANDWIDTH=1500000,RESOLUTION=NaNx720,AUDIO="audio"',
+ "video.m3u8",
+ ].join("\n"),
+ );
+ ffmpeg(
+ [
+ "-f",
+ "lavfi",
+ "-i",
+ "sine=frequency=330:sample_rate=48000",
+ "-t",
+ "5",
+ "-c:a",
+ "libopus",
+ "opus-only.webm",
+ ],
+ root,
+ );
server = Bun.serve({
port: 0,
hostname: "127.0.0.1",
@@ -355,6 +432,103 @@ describe("runLoomImport", () => {
expect(complete?.deferredSourceKey).toBe("user/dash-video/raw-upload.webm");
}, 60_000);
+ test("imports DASH timelines that end with a zero-length entry", async () => {
+ const job = await run(
+ request(
+ "dash-zero",
+ `${base}/media/dash/manifest-zero.mpd?${SIGNATURE}`,
+ {
+ originalUpload: {
+ key: "user/dash-zero/raw-upload.webm",
+ upload: { type: "put", url: `${base}/put/dash-zero/raw.webm` },
+ },
+ },
+ ),
+ );
+ expect(job?.phase).toBe("complete");
+ expect(job?.deferredSourceKey).toBe("user/dash-zero/raw-upload.webm");
+ expect(job?.metadata?.duration).toBeCloseTo(9, 0);
+ }, 60_000);
+
+ test("skips one segment Loom no longer has and keeps later audio in place", async () => {
+ const job = await run(
+ request("hls-gap", `${base}/media/hls-gap/master.m3u8?${SIGNATURE}`),
+ );
+ expect(job?.phase).toBe("complete");
+ const imported = join(root, "gap.mp4");
+ await writeFile(
+ imported,
+ uploads.get("hls-gap/result.mp4") ?? new Uint8Array(),
+ );
+ expect(Number(probe(imported).format.duration)).toBeCloseTo(9, 0);
+ const audioTimes = spawnSync({
+ cmd: [
+ "ffprobe",
+ "-v",
+ "error",
+ "-select_streams",
+ "a:0",
+ "-show_entries",
+ "packet=pts_time",
+ "-of",
+ "csv=p=0",
+ imported,
+ ],
+ })
+ .stdout.toString()
+ .trim()
+ .split("\n")
+ .map(Number);
+ const gaps = audioTimes
+ .slice(1)
+ .map((time, index) => time - (audioTimes[index] ?? time))
+ .filter((delta) => delta > 0.5);
+ expect(gaps).toHaveLength(1);
+ expect(gaps[0]).toBeCloseTo(2, 0);
+ }, 60_000);
+
+ test("fails when Loom is missing more segments than can be skipped", async () => {
+ const job = await run(
+ request("hls-holes", `${base}/media/hls-holes/master.m3u8?${SIGNATURE}`),
+ );
+ expect(job?.phase).toBe("error");
+ expect(job?.error).toContain("Loom is missing 2 of 5 audio segments");
+ }, 60_000);
+
+ test("imports an audio-only HLS recording with a still video track", async () => {
+ const job = await run(
+ request("hls-audio", `${base}/media/hls-audio/master.m3u8?${SIGNATURE}`),
+ );
+ expect(job?.phase).toBe("complete");
+ expect(job?.deferredSourceKey).toBeUndefined();
+ expect(job?.metadata).toMatchObject({
+ width: 1280,
+ height: 720,
+ videoCodec: "h264",
+ audioCodec: "aac",
+ });
+ expect(job?.metadata?.duration).toBeCloseTo(7, 0);
+ expect(uploads.get("hls-audio/thumbnail.jpg")?.byteLength).toBeGreaterThan(
+ 0,
+ );
+ }, 60_000);
+
+ test("imports an Opus-only file as an MP4 with AAC audio", async () => {
+ const job = await run(
+ request("opus-only", `${base}/media/opus-only.webm?${SIGNATURE}`),
+ );
+ expect(job?.phase).toBe("complete");
+ expect(job?.deferredSourceKey).toBeUndefined();
+ expect(job?.metadata).toMatchObject({
+ videoCodec: "h264",
+ audioCodec: "aac",
+ });
+ expect(job?.metadata?.duration).toBeCloseTo(5, 0);
+ expect(uploads.get("opus-only/result.mp4")?.byteLength).toBeGreaterThan(
+ 10_000,
+ );
+ }, 60_000);
+
test("fails VP9 sources that have nowhere to store the original", async () => {
const job = await run(
request("dash-no-target", `${base}/media/dash/manifest.mpd?${SIGNATURE}`),
diff --git a/apps/media-server/src/__tests__/loom-importer/parsing.test.ts b/apps/media-server/src/__tests__/loom-importer/parsing.test.ts
index a210bf23d..66627b1f1 100644
--- a/apps/media-server/src/__tests__/loom-importer/parsing.test.ts
+++ b/apps/media-server/src/__tests__/loom-importer/parsing.test.ts
@@ -11,6 +11,7 @@ import {
resolveFps,
} from "../../loom-importer/media";
import {
+ canSkipMissingSegments,
listMediaPlaylistResources,
parseHlsAttributes,
selectHlsTracks,
@@ -240,3 +241,37 @@ describe("Semaphore", () => {
expect(await semaphore.run(async () => "ok")).toBe("ok");
});
});
+
+describe("canSkipMissingSegments", () => {
+ const segments = (count: number, duration: number) =>
+ Array.from({ length: count }, () => duration);
+
+ test("accepts a source with nothing missing", () => {
+ expect(canSkipMissingSegments([], segments(3, 4))).toBe(true);
+ });
+
+ test("accepts one missing segment in a long recording", () => {
+ expect(canSkipMissingSegments([5], segments(1183, 5))).toBe(true);
+ });
+
+ test("accepts a missing short final segment in a short recording", () => {
+ expect(canSkipMissingSegments([3.07], [...segments(33, 6), 3.07])).toBe(
+ true,
+ );
+ });
+
+ test("rejects more missing segments than one percent allows", () => {
+ expect(canSkipMissingSegments([6, 6], segments(34, 6))).toBe(false);
+ expect(canSkipMissingSegments(segments(12, 5), segments(1183, 5))).toBe(
+ false,
+ );
+ });
+
+ test("rejects a missing segment that is too long to skip", () => {
+ expect(canSkipMissingSegments([20], segments(30, 20))).toBe(false);
+ });
+
+ test("rejects losing every segment", () => {
+ expect(canSkipMissingSegments([2], [2])).toBe(false);
+ });
+});
diff --git a/apps/media-server/src/loom-importer/fetch.ts b/apps/media-server/src/loom-importer/fetch.ts
index bdb4f2b40..8313e1126 100644
--- a/apps/media-server/src/loom-importer/fetch.ts
+++ b/apps/media-server/src/loom-importer/fetch.ts
@@ -2,6 +2,12 @@ import { open, rename, rm } from "node:fs/promises";
export class LoomSourceRejectedError extends Error {}
+export class LoomSegmentStatusError extends Error {
+ constructor(readonly status: number) {
+ super(`Loom segment request failed with ${status}`);
+ }
+}
+
export class Semaphore {
private available: number;
private readonly waiters: Array<() => void> = [];
@@ -127,9 +133,7 @@ export async function downloadToFile(
});
if (!response.ok) {
await response.body?.cancel().catch(() => {});
- const error = new Error(
- `Loom segment request failed with ${response.status}`,
- );
+ const error = new LoomSegmentStatusError(response.status);
if (!RETRYABLE_STATUS.has(response.status)) {
permanentFailures++;
if (permanentFailures >= PERMANENT_STATUS_ATTEMPTS) throw error;
diff --git a/apps/media-server/src/loom-importer/job.ts b/apps/media-server/src/loom-importer/job.ts
index b21c52006..7f3685fd0 100644
--- a/apps/media-server/src/loom-importer/job.ts
+++ b/apps/media-server/src/loom-importer/job.ts
@@ -27,6 +27,7 @@ import {
type Semaphore,
} from "./fetch";
import {
+ buildAudioOnlyArgs,
buildRemuxArgs,
chooseContainer,
PLAYLIST_INPUT_ARGS,
@@ -345,7 +346,10 @@ export async function runLoomImport(
signal,
);
const codecs = await probeCodecs(source.inputArgs, source.input, signal);
- const container = chooseContainer(codecs);
+ const container =
+ codecs.video === null && codecs.audio !== null
+ ? "audio-only"
+ : chooseContainer(codecs);
if (!container)
throw new UnsupportedStreamError(
`Loom source codecs are not supported (${codecs.video ?? "none"}/${codecs.audio ?? "none"})`,
@@ -357,16 +361,24 @@ export async function runLoomImport(
progressUpdate(job.jobId, "processing", 65, "Packaging video...");
outputPath = join(
workDir,
- container === "mp4" ? "result.mp4" : "original.webm",
+ container === "webm" ? "original.webm" : "result.mp4",
);
await runCommand(
- buildRemuxArgs(
- source.inputArgs,
- source.input,
- outputPath,
- container,
- codecs.audio !== null,
- ),
+ container === "audio-only" && codecs.audio
+ ? buildAudioOnlyArgs(
+ source.inputArgs,
+ source.input,
+ outputPath,
+ codecs.audio,
+ codecs.duration,
+ )
+ : buildRemuxArgs(
+ source.inputArgs,
+ source.input,
+ outputPath,
+ container === "webm" ? "webm" : "mp4",
+ codecs.audio !== null,
+ ),
settings.remuxTimeoutMs,
signal,
);
diff --git a/apps/media-server/src/loom-importer/media.ts b/apps/media-server/src/loom-importer/media.ts
index cc0689a50..0fb221ee9 100644
--- a/apps/media-server/src/loom-importer/media.ts
+++ b/apps/media-server/src/loom-importer/media.ts
@@ -76,6 +76,7 @@ interface ProbeOutput {
export interface SourceCodecs {
video: string | null;
audio: string | null;
+ duration: number | null;
}
export async function probeCodecs(
@@ -91,7 +92,7 @@ export async function probeCodecs(
"error",
...inputArgs,
"-show_entries",
- "stream=codec_type,codec_name",
+ "stream=codec_type,codec_name:format=duration",
"-of",
"json",
input,
@@ -101,6 +102,7 @@ export async function probeCodecs(
),
) as ProbeOutput;
const streams = output.streams ?? [];
+ const duration = Number(output.format?.duration);
return {
video:
streams.find((stream) => stream.codec_type === "video")?.codec_name ??
@@ -108,6 +110,7 @@ export async function probeCodecs(
audio:
streams.find((stream) => stream.codec_type === "audio")?.codec_name ??
null,
+ duration: Number.isFinite(duration) && duration > 0 ? duration : null,
};
}
@@ -184,7 +187,9 @@ export async function probeOutput(
export type ImportContainer = "mp4" | "webm";
-export function chooseContainer(codecs: SourceCodecs): ImportContainer | null {
+export function chooseContainer(
+ codecs: Pick<SourceCodecs, "video" | "audio">,
+): ImportContainer | null {
if (
codecs.video === "h264" &&
(codecs.audio === null || codecs.audio === "aac")
@@ -232,3 +237,52 @@ export function buildRemuxArgs(
output,
];
}
+
+export function buildAudioOnlyArgs(
+ inputArgs: string[],
+ input: string,
+ output: string,
+ audioCodec: string,
+ duration: number | null,
+): string[] {
+ return [
+ "ffmpeg",
+ "-hide_banner",
+ "-nostdin",
+ "-loglevel",
+ "error",
+ "-f",
+ "lavfi",
+ "-i",
+ `color=c=black:s=1280x720:r=30${duration ? `:d=${duration.toFixed(3)}` : ""}`,
+ ...inputArgs,
+ "-i",
+ input,
+ "-map",
+ "0:v:0",
+ "-map",
+ "1:a:0",
+ "-shortest",
+ "-dn",
+ "-sn",
+ "-c:v",
+ "libx264",
+ "-preset",
+ "ultrafast",
+ "-tune",
+ "stillimage",
+ "-pix_fmt",
+ "yuv420p",
+ "-threads",
+ "2",
+ ...(audioCodec === "aac"
+ ? ["-c:a", "copy"]
+ : ["-c:a", "aac", "-b:a", "128k"]),
+ "-movflags",
+ "+faststart",
+ "-f",
+ "mp4",
+ "-y",
+ output,
+ ];
+}
diff --git a/apps/media-server/src/loom-importer/streaming.ts b/apps/media-server/src/loom-importer/streaming.ts
index d6935c0e0..f82e4812a 100644
--- a/apps/media-server/src/loom-importer/streaming.ts
+++ b/apps/media-server/src/loom-importer/streaming.ts
@@ -3,6 +3,7 @@ import { extname, join } from "node:path";
import {
type DownloadToFileOptions,
downloadToFile,
+ LoomSegmentStatusError,
type Semaphore,
} from "./fetch";
@@ -187,6 +188,50 @@ export interface PrefetchResult {
segments: number;
}
+const MISSING_SEGMENT_FRACTION = 0.01;
+const MISSING_SEGMENT_SECONDS = 6;
+
+function segmentDuration(lines: string[], lineIndex: number): number {
+ for (let index = lineIndex - 1; index >= 0; index--) {
+ const line = lines[index]?.trim() ?? "";
+ if (line.startsWith("#EXTINF:"))
+ return Number.parseFloat(tagValue(line)) || 0;
+ if (line && !line.startsWith("#")) return 0;
+ }
+ return 0;
+}
+
+function missingSegmentLimit(segments: number): number {
+ return Math.max(1, Math.floor(segments * MISSING_SEGMENT_FRACTION));
+}
+
+export function canSkipMissingSegments(
+ missingDurations: number[],
+ segmentDurations: number[],
+): boolean {
+ if (missingDurations.length === 0) return true;
+ if (missingDurations.length >= segmentDurations.length) return false;
+ const total = segmentDurations.reduce((sum, value) => sum + value, 0);
+ const missing = missingDurations.reduce((sum, value) => sum + value, 0);
+ return (
+ missingDurations.length <= missingSegmentLimit(segmentDurations.length) &&
+ missing <=
+ Math.max(MISSING_SEGMENT_SECONDS, total * MISSING_SEGMENT_FRACTION)
+ );
+}
+
+function removeSegmentLines(lines: string[], lineIndex: number) {
+ lines[lineIndex] = "";
+ for (let index = lineIndex - 1; index >= 0; index--) {
+ const line = lines[index]?.trim() ?? "";
+ if (line.startsWith("#EXTINF:")) {
+ lines[index] = "";
+ return;
+ }
+ if (line && !line.startsWith("#")) return;
+ }
+}
+
export async function prefetchMaterializedHls(
masterPath: string,
workDir: string,
@@ -205,10 +250,12 @@ export async function prefetchMaterializedHls(
const playlists = await Promise.all(
tracks.map(async (track) => {
const content = await readFile(track.path, "utf8");
+ const resources = listMediaPlaylistResources(content);
return {
...track,
lines: content.split(/\r?\n/),
- resources: listMediaPlaylistResources(content),
+ resources,
+ segments: resources.filter((resource) => !resource.attribute).length,
};
}),
);
@@ -229,17 +276,47 @@ export async function prefetchMaterializedHls(
totalSegments: downloads.length,
bytes: 0,
};
+ const missing = new Set<(typeof downloads)[number]>();
+ const playlistBytes = new Map<string, number>();
const results = await Promise.allSettled(
downloads.map((download) =>
options.jobSemaphore
.run(() =>
options.globalSemaphore.run(async () => {
signal.throwIfAborted();
- const bytes = await downloadToFile(
- download.resource.url,
- download.path,
- { ...options, signal },
- );
+ let bytes: number;
+ try {
+ bytes = await downloadToFile(
+ download.resource.url,
+ download.path,
+ { ...options, signal },
+ );
+ } catch (error) {
+ if (
+ !(error instanceof LoomSegmentStatusError) ||
+ error.status !== 404 ||
+ download.resource.attribute
+ )
+ throw error;
+ missing.add(download);
+ progress.completedSegments++;
+ const lost = [...missing].filter(
+ (entry) => entry.playlist === download.playlist,
+ ).length;
+ if (lost > missingSegmentLimit(download.playlist.segments)) {
+ const tooMany = new Error(
+ `Loom is missing ${lost} of ${download.playlist.segments} ${download.playlist.name} segments`,
+ );
+ failure.abort(tooMany);
+ throw tooMany;
+ }
+ return;
+ }
+ if (!download.resource.attribute)
+ playlistBytes.set(
+ download.playlist.name,
+ (playlistBytes.get(download.playlist.name) ?? 0) + bytes,
+ );
progress.bytes += bytes;
progress.completedSegments++;
if (progress.bytes > options.maxBytes) {
@@ -263,7 +340,24 @@ export async function prefetchMaterializedHls(
options.signal.throwIfAborted();
throw failure.signal.reason ?? rejected.reason;
}
+ for (const playlist of playlists) {
+ const media = downloads.filter(
+ (download) =>
+ download.playlist === playlist && !download.resource.attribute,
+ );
+ const durationOf = (download: (typeof downloads)[number]) =>
+ segmentDuration(playlist.lines, download.resource.lineIndex);
+ const lost = media.filter((download) => missing.has(download));
+ if (!canSkipMissingSegments(lost.map(durationOf), media.map(durationOf)))
+ throw new Error(
+ `Loom is missing ${lost.length} of ${media.length} ${playlist.name} segments`,
+ );
+ }
for (const download of downloads) {
+ if (missing.has(download)) {
+ removeSegmentLines(download.playlist.lines, download.resource.lineIndex);
+ continue;
+ }
const line = download.playlist.lines[download.resource.lineIndex] ?? "";
download.playlist.lines[download.resource.lineIndex] = download.resource
.attribute
@@ -277,7 +371,16 @@ export async function prefetchMaterializedHls(
localPlaylists.set(playlist.path, localPath);
}
const localMasterPath = join(workDir, "master-local.m3u8");
- if (selection.masterLines.length > 0) {
+ const audioPlaylist = selection.audio
+ ? localPlaylists.get(selection.audio)
+ : undefined;
+ if (
+ audioPlaylist &&
+ (playlistBytes.get("video") ?? 0) === 0 &&
+ (playlistBytes.get("audio") ?? 0) > 0
+ ) {
+ await writeFile(localMasterPath, await readFile(audioPlaylist, "utf8"));
+ } else if (selection.masterLines.length > 0) {
await writeFile(
localMasterPath,
selection.masterLines