Open source CapCut alternative.
TypeScript
feat(cut): build compat proxies and forced re-encodes [rebuild]
David Leed9fc0fda324
diff --git a/site/src/cut/lib/convertMode.ts b/site/src/cut/lib/convertMode.tsnew file mode 100644index 00000000..7b162968--- /dev/null+++ b/site/src/cut/lib/convertMode.ts@@ -0,0 +1,15 @@+/** How a conversion treats a video stream that already fits its container:+ * "fit" copies it, "reencode" rewrites it from what ffmpeg decodes — for a+ * stream the page parses and then reads no frames from. Shared by the page,+ * the engine, the cloud routes and the worker. */+export const CONVERT_MODES = ["fit", "reencode"] as const;++export type ConvertMode = (typeof CONVERT_MODES)[number];++/** A request's `mode` field: absent is "fit"; anything else unknown is null. */+export function convertModeIn(value: unknown): ConvertMode | null {+ if (value === undefined) {+ return "fit";+ }+ return CONVERT_MODES.find((m) => m === value) ?? null;+}diff --git a/site/src/cut/lib/engineFeatures.ts b/site/src/cut/lib/engineFeatures.tsindex 929e46f8..3b96519f 100644--- a/site/src/cut/lib/engineFeatures.ts+++ b/site/src/cut/lib/engineFeatures.ts@@ -4,6 +4,6 @@* hands an engine work an older app would silently drop. A build from before* the list existed answers with none.*/-export const ENGINE_FEATURES = ["export.range", "export.name", "export.prores4444", "color.v2", "media.proxy", "library.proxy", "export.hdr", "doc.timelines", "doc.timelineAspect", "export.loudness", "export.stems", "export.sound"] as const;+export const ENGINE_FEATURES = ["export.range", "export.name", "export.prores4444", "color.v2", "media.proxy", "library.proxy", "export.hdr", "doc.timelines", "doc.timelineAspect", "export.loudness", "export.stems", "export.sound", "media.compat"] as const;export type EngineFeature = (typeof ENGINE_FEATURES)[number];diff --git a/site/src/cut/lib/proxyNames.test.ts b/site/src/cut/lib/proxyNames.test.tsindex e6395ead..28ced252 100644--- a/site/src/cut/lib/proxyNames.test.ts+++ b/site/src/cut/lib/proxyNames.test.ts@@ -1,5 +1,5 @@import { expect, test } from "bun:test";-import { isLightProxyName, lightProxyNameFor } from "./proxyNames";+import { compatCoverOf, compatCovers, compatProxyNameFor, isLightProxyName, lightProxyNameFor } from "./proxyNames";/** Two masters that share a stem (clip.mov, clip.mp4) are different footage,* so their light proxies keep the whole master name apart. */@@ -10,3 +10,23 @@ test("a light proxy is named after the whole master name", () => {expect(isLightProxyName("clip.mov.light-1.mp4")).toBe(true);expect(isLightProxyName("clip.proxy.mp4")).toBe(false);});++/** A compat proxy says in its name what it carries: VP9 picture in WebM, or+ * the master's picture with Opus sound in MP4. */+test("a compat proxy names what it covers", () => {+ expect(compatProxyNameFor("clip.mov", "video", false)).toBe("clip.mov.compat.webm");+ expect(compatProxyNameFor("clip.mov", "audio", true)).toBe("clip.mov.compat.light.mp4");+ expect(compatCoverOf("clip.mov.compat.webm")).toBe("video");+ expect(compatCoverOf("clip.mov.compat.light-1.mp4")).toBe("audio");+ expect(compatCoverOf("clip.mov.light.mp4")).toBeNull();+ expect(compatCoverOf("clip.proxy.mp4")).toBeNull();+ // A light compat proxy serves a heavy video's stream as well.+ expect(isLightProxyName("clip.mov.compat.light.webm")).toBe(true);+ expect(isLightProxyName("clip.mov.compat.webm")).toBe(false);+});++test("a VP9 compat proxy covers sound too", () => {+ expect(compatCovers("clip.mov.compat.webm", "audio")).toBe(true);+ expect(compatCovers("clip.mov.compat.mp4", "video")).toBe(false);+ expect(compatCovers("clip.proxy.mp4", "audio")).toBe(false);+});diff --git a/site/src/cut/lib/proxyNames.ts b/site/src/cut/lib/proxyNames.tsindex 33aeb004..97ae0397 100644--- a/site/src/cut/lib/proxyNames.ts+++ b/site/src/cut/lib/proxyNames.ts@@ -13,8 +13,49 @@export const lightProxyNameFor = (fileName: string) => `${fileName}.light.mp4`;/** Whether a proxy's file name is a light proxy's (a deduped `.light-1.mp4`- * included). */-export const isLightProxyName = (fileName: string) => /\.light(-\d+)?\.mp4$/.test(fileName);+ * included). A light compat proxy is one too. */+export const isLightProxyName = (fileName: string) => /\.light(-\d+)?\.(mp4|webm)$/.test(fileName);++/**+ * The compat proxy: the master with each stream this browser cannot decode+ * written in a codec every browser does (codecSupport.ts). "video" is VP9+ * picture with Opus sound in WebM, for a browser without H.264; "audio" keeps+ * the master's picture and writes Opus sound in MP4, for a browser whose AAC+ * decoder fails. A light one is capped like a heavy video's light proxy.+ */+export type CompatCover = "audio" | "video";++/** The compat proxy's file name beside its master. */+export const compatProxyNameFor = (fileName: string, cover: CompatCover, light: boolean) =>+ `${fileName}.compat${light ? ".light" : ""}.${cover === "video" ? "webm" : "mp4"}`;++/** A request's `compat` field as a cover, or null when it names none. */+export const compatCoverIn = (value: unknown): CompatCover | null =>+ value === "audio" || value === "video" ? value : null;++/** Where the preview reads `need` from: the compat proxy when it covers that+ * stream, else null for the master. */+export function compatUrl(asset: { proxy?: { fileName: string }; proxyUrl?: string }, need: CompatCover): string | null {+ if (!asset.proxy || !asset.proxyUrl) {+ return null;+ }+ return compatCovers(asset.proxy.fileName, need) ? asset.proxyUrl : null;+}++/** What a compat proxy covers, by its name (a deduped `-1` included); null for+ * any other proxy. */+export function compatCoverOf(fileName: string): CompatCover | null {+ const m = /\.compat(\.light)?(-\d+)?\.(mp4|webm)$/.exec(fileName);+ if (!m) return null;+ return m[3] === "webm" ? "video" : "audio";+}++/** Whether the proxy named `fileName` serves a browser missing `need`: a VP9+ * proxy carries Opus sound as well. */+export function compatCovers(fileName: string, need: CompatCover): boolean {+ const cover = compatCoverOf(fileName);+ return cover === "video" || (cover !== null && cover === need);+}/** What a failed proxy job's row records in `result.fault` when ffmpeg ran and* refused the master: the file's own fault, which the cloud route holds offdiff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex 231f4b11..9599383e 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -612,7 +612,7 @@ Project media (\`media\` in editor_state) is every file in the open project. The- file_asset moves a created asset into Media (to:"media", clears its origin) or copies any asset into the Library (to:"library").- media_organize keeps the Media panel's own folders (\`mediaFolders\` in editor_state, per project): create/rename/delete a folder (a delete removes every file in it, timeline clips and all), move_asset to file media into one — filing a created asset also moves it into Media, like dropping its card on a folder tile. rename_asset renames any project file.- delete_asset removes an asset from the project plus its timeline clips; the media does not come back with undo, so only on an explicit ask, and say what went with it.-- convert_media rewrites project media as MP4 (H.264/AAC) in place, one call for a whole list of asset ids. Reach for it when footage plays black or shows a format warning — phone HEVC, ProRes, raw PCM sound — and whenever someone asks for mp4 files. That ask is usually files dropped straight on this chat: convert them and each one is on its own card in the thread, converted, ready to download or drag onto the timeline — close by saying they're on their cards. An asset that came from the Media panel stays there. Streams that already fit are copied, so most .mov files convert in seconds, and a file that is already H.264/AAC MP4 comes back untouched (say so instead of claiming a conversion); max_height shrinks an oversized source at the same time. Clips already cut from the asset keep playing. Conversion runs through the project backend, including headless tools. A browser missing a required codec uses hosted preparation and saves the result back into its own project.+- convert_media rewrites project media as MP4 (H.264/AAC) in place, one call for a whole list of asset ids. Reach for it when footage plays black or shows a format warning — phone HEVC, ProRes, raw PCM sound — and whenever someone asks for mp4 files. That ask is usually files dropped straight on this chat: convert them and each one is on its own card in the thread, converted, ready to download or drag onto the timeline — close by saying they're on their cards. An asset that came from the Media panel stays there. Streams that already fit are copied, so most .mov files convert in seconds, and a file that is already H.264/AAC MP4 comes back untouched (say so instead of claiming a conversion) — mode reencode rewrites H.264 that still plays black; max_height shrinks an oversized source at the same time. Clips already cut from the asset keep playing. Conversion runs through the project backend, including headless tools. A browser missing a required codec uses hosted preparation and saves the result back into its own project.Preview selection is shared with the timeline. Use select with additive:true to toggle multiple items, then move_selection with dx/dy frame fractions to move them together. Caption positions apply to their whole track.The Library is shared across every project: folders, reusable assets, and templates — a template is a saved arrangement (clips, overlays, titles, captions, by reference) that comes back editable.diff --git a/site/src/cut/server/ai/headlessTools.ts b/site/src/cut/server/ai/headlessTools.tsindex 8822e01f..05792b8e 100644--- a/site/src/cut/server/ai/headlessTools.ts+++ b/site/src/cut/server/ai/headlessTools.ts@@ -93,7 +93,7 @@ setHostMediaStore({const fileName = await uniqueName(mp4NameFor(source.fileName), (n) => mediaPath(projectId, n));const outPath = mediaPath(projectId, fileName);const handle = { tmpDir: "", outPath, progress: 0, log: [] as string[] };- const outcome = await convertToMp4(handle, src, outPath, { maxHeight: opts.maxHeight });+ const outcome = await convertToMp4(handle, src, outPath, { maxHeight: opts.maxHeight, mode: opts.mode });return {fileName: outcome.unchanged ? source.fileName : fileName,...(outcome.unchanged ? { unchanged: true } : {}),diff --git a/site/src/cut/server/cloud/jobs.ts b/site/src/cut/server/cloud/jobs.tsindex 97a9de42..2e139348 100644--- a/site/src/cut/server/cloud/jobs.ts+++ b/site/src/cut/server/cloud/jobs.ts@@ -12,9 +12,10 @@ import { MEDIA_REDIRECT_HEADERS, mediaObjectUrl, mediaUrlLifetime } from "./mediimport { del, head, overlayKey, overlayPrefix, presignPut, projectExportKey } from "./r2";import { contentTypeFor } from "../serveFile";import { containerOfName, deliveryContainer, exportBaseName, jobWantsStems, specMediaFiles, STEMS_MIME, stemsArchiveName, type ExportContainer, type ExportFiles } from "../../lib/exportDelivery";+import { convertModeIn } from "../../lib/convertMode";import { LOUDNESS_IDS, type LoudnessId } from "../../lib/loudnessSettings";import type { DocExportAudio } from "../../lib/exportPresets";-import { PROXY_SOURCE_FAULT } from "../../lib/proxyNames";+import { compatCoverIn, compatCovers, isLightProxyName, PROXY_SOURCE_FAULT, type CompatCover } from "../../lib/proxyNames";import { addUsage, outputByteCeiling, quotaCheck } from "./usage";import { caught, err, redirect } from "./util";@@ -24,6 +25,34 @@ const HOUR_MS = 60 * 60 * 1000;* with (server/proxy.ts `proxyNameFor`). */const proxyStem = (fileName: string) => fileName.replace(/\.[^./\\]+$/, "") || "media";+/** The newest proxy of `file` already built that answers an ask: light or+ * full quality, and the compat cover it names (lib/proxyNames.ts). */+async function builtProxy(userId: string, projectId: string, file: string, light: boolean, compat: CompatCover | null) {+ // A compat proxy is named from the whole master name, as a light one is.+ let prefix = light ? `${file}.light` : `${proxyStem(file)}.proxy`;+ if (compat) {+ prefix = light ? `${file}.compat.light` : `${file}.compat`;+ }+ const rows = await prisma.cutMediaObject.findMany({+ where: { userId, projectId, kind: "proxy", uploadState: "complete", fileName: { startsWith: prefix } },+ select: { fileName: true, bytes: true },+ orderBy: { createdAt: "desc" },+ });+ if (!compat) {+ return rows[0] ?? null;+ }++ // `clip.mp4.compat` also starts `clip.mp4.compat.light`: a full-size ask+ // takes a full-size proxy that covers the stream.+ return rows.find((r) => compatCovers(r.fileName, compat) && isLightProxyName(r.fileName) === light) ?? null;+}++/** Whether a proxy job's spec asked for what this request asks. */+function sameProxyAsk(spec: unknown, light: boolean, compat: CompatCover | null): boolean {+ const s = (spec ?? {}) as { maxHeight?: number; compat?: unknown };+ return (s.maxHeight !== undefined) === light && compatCoverIn(s.compat) === compat;+}+/** How long finished jobs stay in the export-jobs feed — the engine's registry* keeps a bounded terminal backlog; the cloud keeps a day. */const FEED_WINDOW_MS = 24 * 60 * 60 * 1000;@@ -926,8 +955,10 @@ export const jobsCloud = {* files as MP4 and registers it beside the original. */async convert(userId: string, projectId: string, req: Request) {try {- const { file, maxHeight } = (await req.json()) as { file?: string; maxHeight?: number };+ const { file, maxHeight, mode: askedMode } = (await req.json()) as { file?: string; maxHeight?: number; mode?: unknown };if (!file) return err("file is required.", 400);+ const mode = convertModeIn(askedMode);+ if (!mode) return err("mode is fit or reencode.", 400);if (!(await getProject(userId, projectId))) return err("Project not found.", 404);const over = await quotaCheck(userId, 0, EXPORT_QUOTA_MARGIN);if (over) return over;@@ -943,7 +974,7 @@ export const jobsCloud = {userId,projectId,kind: "convert",- spec: { file, ...(maxHeight ? { maxHeight } : {}) } as unknown as Prisma.InputJsonValue,+ spec: { file, ...(maxHeight ? { maxHeight } : {}), ...(mode === "reencode" ? { mode } : {}) } as unknown as Prisma.InputJsonValue,},});wakeRenderWorker();@@ -957,11 +988,14 @@ export const jobsCloud = {* smaller `maxHeight` it asks for: the worker builds it and* registers it beside the master as a quota-exempt object. A proxy already* built answers at once, and a job already on its way answers with that- * job, so a project copied to the cloud and then opened queues one. */+ * job, so a project copied to the cloud and then opened queues one.+ * `compat` asks for the compat proxy (lib/proxyNames.ts). */async proxy(userId: string, projectId: string, req: Request) {try {- const { file, maxHeight: asked } = (await req.json()) as { file?: string; maxHeight?: unknown };+ const { file, maxHeight: asked, compat: askedCompat } = (await req.json()) as { file?: string; maxHeight?: unknown; compat?: unknown };if (!file) return err("file is required.", 400);+ const compat = compatCoverIn(askedCompat);+ if (askedCompat !== undefined && !compat) return err("compat is audio or video.", 400);const maxHeight = typeof asked === "number" && Number.isFinite(asked) ? Math.round(Math.min(4320, Math.max(360, asked))) : undefined;if (!(await getProject(userId, projectId))) return err("Project not found.", 404);const source = await prisma.cutMediaObject.findFirst({@@ -972,17 +1006,13 @@ export const jobsCloud = {// A heavy video's request (it names a height) wants the light proxy// (lib/proxyNames.ts); a full-quality one built earlier doesn't answer it.const light = maxHeight !== undefined;- const built = await prisma.cutMediaObject.findFirst({- where: { userId, projectId, kind: "proxy", uploadState: "complete", fileName: { startsWith: light ? `${file}.light` : `${proxyStem(file)}.proxy` } },- select: { fileName: true, bytes: true },- orderBy: { createdAt: "desc" },- });+ const built = await builtProxy(userId, projectId, file, light, compat);if (built) return Response.json({ fileName: built.fileName, sizeBytes: Number(built.bytes) });const runningRows = await prisma.cutRenderJob.findMany({where: { userId, projectId, kind: "proxy", state: { in: ["queued", "running"] }, spec: { path: ["file"], equals: file } },select: { id: true, spec: true },});- const running = runningRows.find((r) => ((r.spec as { maxHeight?: number } | null)?.maxHeight !== undefined) === light);+ const running = runningRows.find((r) => sameProxyAsk(r.spec, light, compat));if (running) {wakeRenderWorker();return Response.json({ jobId: running.id });@@ -1002,13 +1032,13 @@ export const jobsCloud = {},select: { spec: true },});- if (failedRows.some((r) => ((r.spec as { maxHeight?: number } | null)?.maxHeight !== undefined) === light)) {+ if (failedRows.some((r) => sameProxyAsk(r.spec, light, compat))) {return err("The preview proxy could not be made from this file lately; the original plays instead.", 409);}const capped = await renderJobCheck(userId);if (capped) return capped;const row = await prisma.cutRenderJob.create({- data: { userId, projectId, kind: "proxy", spec: { file, ...(maxHeight ? { maxHeight } : {}) } as unknown as Prisma.InputJsonValue },+ data: { userId, projectId, kind: "proxy", spec: { file, ...(maxHeight ? { maxHeight } : {}), ...(compat ? { compat } : {}) } as unknown as Prisma.InputJsonValue },});wakeRenderWorker();return Response.json({ jobId: row.id });diff --git a/site/src/cut/server/cloud/library.ts b/site/src/cut/server/cloud/library.tsindex 3a7f5be2..0f644892 100644--- a/site/src/cut/server/cloud/library.ts+++ b/site/src/cut/server/cloud/library.ts@@ -25,6 +25,7 @@ import { renderJobCheck } from "./limits";import { wakeRenderWorker } from "./wake";import { libraryTypeOf } from "@/cut/lib/libraryFileType";import { itemName } from "@/cut/lib/itemName";+import { compatCoverIn, compatCovers } from "@/cut/lib/proxyNames";import type { AssetType } from "@/cut/lib/types";import { caught, decodeFileParam, dedupeName, err, HttpResponseError, inspirationFolderId, redirect, safeFileName, storableFileName } from "./util";@@ -701,16 +702,21 @@ export const libraryCloud = {/** Queue the playable copy of a library video the browser cannot decode.* A row that already names one answers with itself, and a job already on- * its way answers with that job. */- async proxy(userId: string, id: string) {+ * its way answers with that job. `compat` asks for the compat proxy+ * (lib/proxyNames.ts), which an audio file can take as well. */+ async proxy(userId: string, id: string, req: Request) {try {+ const { compat: asked } = (await req.json().catch(() => ({}))) as { compat?: unknown };+ const compat = compatCoverIn(asked);+ if (asked !== undefined && !compat) return err("compat is audio or video.", 400);const asset = await prisma.cutLibraryAsset.findFirst({ where: { id, userId, deletedAt: null } });if (!asset) return err("Library asset not found.", 404);const obj = await prisma.cutMediaObject.findFirst({ where: { id: asset.mediaObjectId, userId } });if (!obj) return err("Library asset not found.", 404);const view = assetView(asset, obj);- if (view.type !== "video") return err("Only a video takes a playable copy.", 400);- if (view.proxyFile) return Response.json(view);+ const takes = view.type === "video" || (compat === "audio" && view.type === "audio");+ if (!takes) return err("Only a video takes a playable copy.", 400);+ if (view.proxyFile && (!compat || compatCovers(view.proxyFile, compat))) return Response.json(view);const jobId = libraryProxyJobId(id);const existing = await prisma.cutRenderJob.findFirst({ where: { id: jobId, userId } });if (existing && (existing.state === "queued" || existing.state === "running")) {@@ -719,7 +725,7 @@ export const libraryCloud = {}const capped = await renderJobCheck(userId);if (capped) return capped;- const spec = { target: "library", assetId: id } as Prisma.InputJsonValue;+ const spec = { target: "library", assetId: id, ...(compat ? { compat } : {}) } as Prisma.InputJsonValue;if (existing) {// A finished or failed run goes again under the same id.await prisma.cutRenderJob.updateMany({diff --git a/site/src/cut/server/cloud/routes.ts b/site/src/cut/server/cloud/routes.tsindex 3ff4b177..4ec65cc4 100644--- a/site/src/cut/server/cloud/routes.ts+++ b/site/src/cut/server/cloud/routes.ts@@ -97,7 +97,7 @@ const CUT_CLOUD_ROUTES: CloudRoute[] = [{ method: "PUT", path: "/api/cut-cloud/library/folders/:id", handler: (r, u, p) => libraryCloud.updateFolder(u, p.id, r) },{ method: "DELETE", path: "/api/cut-cloud/library/folders/:id", handler: (_r, u, p) => libraryCloud.deleteFolder(u, p.id) },{ method: "GET", path: "/api/cut-cloud/library/media/:file", handler: (r, u, p) => libraryCloud.serveMedia(u, p.file, wantsDownload(r)) },- { method: "POST", path: "/api/cut-cloud/library/:id/proxy", handler: (_r, u, p) => libraryCloud.proxy(u, p.id) },+ { method: "POST", path: "/api/cut-cloud/library/:id/proxy", handler: (r, u, p) => libraryCloud.proxy(u, p.id, r) },{ method: "PUT", path: "/api/cut-cloud/library/:id", handler: (r, u, p) => libraryCloud.updateMeta(u, p.id, r) },{ method: "DELETE", path: "/api/cut-cloud/library/:id", handler: (_r, u, p) => libraryCloud.remove(u, p.id) },diff --git a/site/src/cut/server/convert.test.ts b/site/src/cut/server/convert.test.tsindex 020c1b07..d337ed80 100644--- a/site/src/cut/server/convert.test.ts+++ b/site/src/cut/server/convert.test.ts@@ -64,6 +64,28 @@ describe("import conversion", () => {expect(await convertToMp4(handle(path.join(dir, "unused.mp4")), src, path.join(dir, "unused.mp4"))).toMatchObject({ unchanged: true });});+ test("a forced re-encode rewrites an already playable MP4", async () => {+ // A stream that parses and then yields no frames in the page — a missing+ // keyframe, a broken reference chain — is rewritten from what ffmpeg+ // decodes, so the copy starts on a clean keyframe.+ const src = path.join(dir, "frameless.mp4");+ const out = path.join(dir, "rewritten.mp4");+ ffmpeg(["-f", "lavfi", "-i", "color=s=64x64:r=10:d=0.2", "-c:v", "libx264", src]);+ expect(await convertToMp4(handle(out), src, out, { mode: "reencode" })).toMatchObject({ transcodedVideo: true });+ expect(await streamCodecs(out)).toMatchObject({ video: "h264" });+ expect((await readFile(out)).equals(await readFile(src))).toBe(false);+ });++ test("a file named .mp4 in another container is remuxed", async () => {+ // A page whose demuxer can't parse the file hands it here; the name alone+ // must not send it back unchanged.+ const src = path.join(dir, "mislabeled.mp4");+ const out = path.join(dir, "remuxed.mp4");+ ffmpeg(["-f", "lavfi", "-i", "color=s=64x64:r=10:d=0.2", "-c:v", "libx264", "-f", "matroska", src]);+ expect(await convertToMp4(handle(out), src, out)).toMatchObject({ transcodedVideo: false });+ expect(execFileSync("ffprobe", ["-v", "error", "-show_entries", "format=format_name", "-of", "csv=p=0", out]).toString()).toContain("mp4");+ });+test("audio-only phone media becomes playable AAC without a video stream", async () => {const src = path.join(dir, "recording.mov");const out = path.join(dir, "recording.m4a");diff --git a/site/src/cut/server/convert.ts b/site/src/cut/server/convert.tsindex 5dd917ea..ed32447b 100644--- a/site/src/cut/server/convert.ts+++ b/site/src/cut/server/convert.ts@@ -1,6 +1,7 @@import { spawn } from "node:child_process";import { h264Encoder, runFfmpeg, vtQuality, type RenderHandle } from "./exportPipeline";import { assertGraphSafe } from "./filterGraph";+import type { ConvertMode } from "../lib/convertMode";import { videoDimensions } from "./util";/**@@ -47,10 +48,11 @@ const MP4_EXT = /\.(mp4|m4v)$/i;* to leave it alone. */export const fitsAlready = (file: string,- codecs: { video?: string; audio?: string; remux?: boolean; videoNeedsEncoding?: boolean },+ codecs: { video?: string; audio?: string; remux?: boolean; videoNeedsEncoding?: boolean; mp4?: boolean },work: { shrink: boolean }) =>MP4_EXT.test(file) &&+ !!codecs.mp4 &&!work.shrink &&!codecs.remux &&!codecs.videoNeedsEncoding &&@@ -66,8 +68,9 @@ export type MediaStream = {disposition?: { default?: number; attached_pic?: number };};-/** Choose the playable audio rendition carried beside optional spatial tracks. */-export function selectConversionStreams(streams: MediaStream[]) {+/** Choose the playable audio rendition carried beside optional spatial tracks.+ * `formatName` is ffprobe's name for the container, e.g. "mov,mp4,m4a,3gp,3g2,mj2". */+export function selectConversionStreams(streams: MediaStream[], formatName = "") {const video = streams.find((s) => s.codec_type === "video" && !s.disposition?.attached_pic);const audioTracks = streams.filter((s) => s.codec_type === "audio");const knownAudio = audioTracks.filter((s) => s.codec_name && s.codec_name !== "unknown" && s.codec_tag_string !== "apac");@@ -81,6 +84,9 @@ export function selectConversionStreams(streams: MediaStream[]) {videoNeedsEncoding: !!video && (video.codec_name !== "h264" || (!!video.pix_fmt && video.pix_fmt !== "yuv420p" && video.pix_fmt !== "yuvj420p")),audioIndex: audio?.index,remux: audioTracks.length > 1,+ // What the bytes are, whatever the name says: a page that can't parse a+ // file named .mp4 hands it here, and only a remux makes it readable.+ mp4: formatName.split(",").includes("mp4"),};}@@ -89,7 +95,7 @@ export function streamCodecs(file: string): Promise<ReturnType<typeof selectConvreturn new Promise((resolve, reject) => {const p = spawn("ffprobe", ["-v", "error", "-show_entries",- "stream=index,codec_type,codec_name,codec_tag_string,pix_fmt:stream_disposition=default,attached_pic",+ "format=format_name:stream=index,codec_type,codec_name,codec_tag_string,pix_fmt:stream_disposition=default,attached_pic","-of", "json", file,]);let out = "";@@ -103,7 +109,8 @@ export function streamCodecs(file: string): Promise<ReturnType<typeof selectConvclearTimeout(timer);if (code !== 0) return reject(new Error("Could not inspect this media file."));try {- resolve(selectConversionStreams((JSON.parse(out).streams ?? []) as MediaStream[]));+ const probe = JSON.parse(out) as { streams?: MediaStream[]; format?: { format_name?: string } };+ resolve(selectConversionStreams(probe.streams ?? [], probe.format?.format_name));} catch (error) { reject(error); }});p.on("error", (error) => { clearTimeout(timer); reject(error); });@@ -119,16 +126,17 @@ export async function convertToMp4(handle: RenderHandle,src: string,out: string,- opts: { maxHeight?: number; onProgress?: (seconds: number) => void } = {}+ opts: { maxHeight?: number; mode?: ConvertMode; onProgress?: (seconds: number) => void } = {}): Promise<ConvertOutcome> {const [codecs, dims] = await Promise.all([streamCodecs(src), videoDimensions(src)]);if (!codecs.video && !codecs.audio) throw new Error("That file carries no video or audio.");const cap = opts.maxHeight && opts.maxHeight > 0 ? Math.round(opts.maxHeight) : 0;const shrink = !!(cap && dims && dims.height > cap);- const transcodedVideo = !!codecs.video && (codecs.videoNeedsEncoding || shrink);+ const forced = opts.mode === "reencode";+ const transcodedVideo = !!codecs.video && (codecs.videoNeedsEncoding || shrink || forced);const transcodedAudio = !!codecs.audio && !KEEPS_AUDIO.has(codecs.audio);- if (fitsAlready(src, codecs, { shrink })) {+ if (!forced && fitsAlready(src, codecs, { shrink })) {return {transcodedVideo: false,transcodedAudio: false,diff --git a/site/src/cut/server/http/convert.ts b/site/src/cut/server/http/convert.tsindex 6c0a06fa..f92dfb92 100644--- a/site/src/cut/server/http/convert.ts+++ b/site/src/cut/server/http/convert.ts@@ -1,6 +1,7 @@import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";import os from "node:os";import path from "node:path";+import { convertModeIn } from "../../lib/convertMode";import { convertToMp4, mp4NameFor } from "../convert";import { errorMessage } from "../util";@@ -21,12 +22,17 @@ export const convertApi = {return Response.json({ error: "file is required." }, { status: 400 });}const maxHeight = Number(form.get("maxHeight"));+ const mode = convertModeIn(form.get("mode") ?? undefined);+ if (!mode) {+ return Response.json({ error: "mode is fit or reencode." }, { status: 400 });+ }const src = path.join(tmp, path.basename(file.name) || "source");const out = path.join(tmp, mp4NameFor(path.basename(file.name) || "media"));await writeFile(src, new Uint8Array(await file.arrayBuffer()));const handle = { tmpDir: tmp, outPath: out, progress: 0, log: [] as string[] };const outcome = await convertToMp4(handle, src, out, {maxHeight: Number.isFinite(maxHeight) && maxHeight > 0 ? maxHeight : undefined,+ mode,});const bytes = await readFile(outcome.unchanged ? src : out);return new Response(new Uint8Array(bytes), {diff --git a/site/src/cut/server/http/projects.ts b/site/src/cut/server/http/projects.tsindex 773dc98d..6f435281 100644--- a/site/src/cut/server/http/projects.ts+++ b/site/src/cut/server/http/projects.ts@@ -1,5 +1,6 @@import { sanitizeGuideLines, sanitizeGuides } from "@/cut/lib/guides";import { execFile } from "node:child_process";+import { compatCoverIn, compatProxyNameFor } from "@/cut/lib/proxyNames";import { isTimelineId, normalizeAspect, sanitizeTimelines, type ProjectDoc } from "@/cut/lib/types";import { detectSilence, extractAudio, makeFreezeFrame, probeDims, probeDuration } from "../frames";import {@@ -25,7 +26,8 @@ import {writeProject,} from "../projects";import { convertToMp4, mp4NameFor } from "../convert";-import { makeProxy, proxyNameFor } from "../proxy";+import { convertModeIn } from "@/cut/lib/convertMode";+import { makeCompatProxy, makeProxy, proxyNameFor } from "../proxy";import { serveFileRange, wantsDownload } from "../serveFile";import { importUrlToProject } from "../urlImport";import { createTranscribeJob, getTranscribeJob, type TranscribeSpec } from "../transcribe";@@ -433,8 +435,10 @@ export const projectsApi = {async convert(req: Request, { id }: { id: string }) {try {if (!(await readProject(id))) return err("Project not found.", 404);- const body = (await req.json()) as { file?: string; maxHeight?: number };+ const body = (await req.json()) as { file?: string; maxHeight?: number; mode?: unknown };if (!body.file) return err("file is required.", 400);+ const mode = convertModeIn(body.mode);+ if (!mode) return err("mode is fit or reencode.", 400);const src = mediaPath(id, body.file);if (!(await exists(src))) return err("Media file missing from project.", 404);const fileName = await uniqueName(mp4NameFor(body.file), (n) => mediaPath(id, n));@@ -442,6 +446,7 @@ export const projectsApi = {const handle = { tmpDir: "", outPath, progress: 0, log: [] as string[] };const outcome = await convertToMp4(handle, src, outPath, {maxHeight: typeof body.maxHeight === "number" ? body.maxHeight : undefined,+ mode,});// Nothing was written for a file that already fits; the project keeps// the one it has.@@ -459,19 +464,23 @@ export const projectsApi = {/** Build the preview proxy of a ProRes master in this project (see* server/proxy.ts). The proxy lands beside the master; writing it onto the* asset is the client's call. The page sends the size and quality it was- * configured with. */+ * configured with, and `compat` for a browser that cannot decode one of+ * the master's streams (lib/proxyNames.ts). */async proxy(req: Request, { id }: { id: string }) {try {if (!(await readProject(id))) return err("Project not found.", 404);- const body = (await req.json()) as { file?: string; maxHeight?: number; crf?: number };+ const body = (await req.json()) as { file?: string; maxHeight?: number; crf?: number; compat?: unknown };if (!body.file) return err("file is required.", 400);if (typeof body.maxHeight !== "number" || typeof body.crf !== "number") return err("maxHeight and crf are required.", 400);+ const compat = compatCoverIn(body.compat);+ if (body.compat !== undefined && !compat) return err("compat is audio or video.", 400);const src = mediaPath(id, body.file);if (!(await exists(src))) return err("Media file missing from project.", 404);- const fileName = await uniqueName(proxyNameFor(body.file), (n) => mediaPath(id, n));+ const fileName = await uniqueName(compat ? compatProxyNameFor(body.file, compat, false) : proxyNameFor(body.file), (n) => mediaPath(id, n));const outPath = mediaPath(id, fileName);const handle = { tmpDir: "", outPath, progress: 0, log: [] as string[] };- const outcome = await makeProxy(handle, src, outPath, { maxHeight: body.maxHeight, crf: body.crf });+ const opts = { maxHeight: body.maxHeight, crf: body.crf };+ const outcome = compat ? await makeCompatProxy(handle, src, outPath, { ...opts, cover: compat }) : await makeProxy(handle, src, outPath, opts);return Response.json({ fileName, ...outcome });} catch (e) {return caught(e, "Could not make the preview proxy.");diff --git a/site/src/cut/server/http/proxy.ts b/site/src/cut/server/http/proxy.tsindex e07f516c..942426be 100644--- a/site/src/cut/server/http/proxy.ts+++ b/site/src/cut/server/http/proxy.ts@@ -1,7 +1,8 @@import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";import os from "node:os";import path from "node:path";-import { makeProxy, proxyNameFor } from "../proxy";+import { compatCoverIn, compatProxyNameFor } from "@/cut/lib/proxyNames";+import { makeCompatProxy, makeProxy, proxyNameFor } from "../proxy";import { errorMessage } from "../util";/**@@ -26,15 +27,26 @@ export const proxyApi = {if (!Number.isFinite(maxHeight) || !Number.isFinite(crf)) {return Response.json({ error: "maxHeight and crf are required." }, { status: 400 });}+ // A browser that cannot decode one of the master's streams asks for+ // the compat proxy (lib/proxyNames.ts).+ const asked = form.get("compat");+ const compat = compatCoverIn(asked);+ if (asked !== null && !compat) {+ return Response.json({ error: "compat is audio or video." }, { status: 400 });+ }const name = path.basename(file.name) || "source";const src = path.join(tmp, name);- const out = path.join(tmp, proxyNameFor(name));+ const out = path.join(tmp, compat ? compatProxyNameFor(name, compat, false) : proxyNameFor(name));await writeFile(src, new Uint8Array(await file.arrayBuffer()));const handle = { tmpDir: tmp, outPath: out, progress: 0, log: [] as string[] };- await makeProxy(handle, src, out, { maxHeight, crf });+ if (compat) {+ await makeCompatProxy(handle, src, out, { maxHeight, crf, cover: compat });+ } else {+ await makeProxy(handle, src, out, { maxHeight, crf });+ }const bytes = await readFile(out);return new Response(new Uint8Array(bytes), {- headers: { "Content-Type": "video/mp4", "Content-Length": String(bytes.length) },+ headers: { "Content-Type": compat === "video" ? "video/webm" : "video/mp4", "Content-Length": String(bytes.length) },});} catch (e) {return Response.json(diff --git a/site/src/cut/server/proxy.test.ts b/site/src/cut/server/proxy.test.tsindex 0d9ace17..e8920ef8 100644--- a/site/src/cut/server/proxy.test.ts+++ b/site/src/cut/server/proxy.test.ts@@ -4,7 +4,7 @@ import { mkdtemp, readFile, rm } from "node:fs/promises";import os from "node:os";import path from "node:path";import { fileURLToPath } from "node:url";-import { makeProxy, probeProxySource, proxyFfmpegArgs, proxyMatrixOf, proxyNameFor } from "./proxy";+import { makeCompatProxy, makeProxy, probeProxySource, proxyFfmpegArgs, proxyMatrixOf, proxyNameFor } from "./proxy";import { openMedia, probeMediaFile, videoTrackOf } from "@/cut/lib/mediaRead";const FIXTURE = path.join(path.dirname(fileURLToPath(import.meta.url)), "../lib/fixtures/prores-422hq-10bit.mov");@@ -156,3 +156,54 @@ describe("proxy", () => {expect(bytes.indexOf("moov")).toBeLessThan(bytes.indexOf("mdat"));}, 60_000);});++describe("compat proxy", () => {+ /** Codec names of `file`'s streams, video first. */+ const codecsOf = (file: string) =>+ ffprobe(["-show_entries", "stream=codec_type,codec_name", "-of", "csv=p=0", file]).trim().split("\n").sort();+ /** A one-second phone-like clip: H.264 High picture, AAC sound. */+ const phoneClip = (name: string, colour: string[] = []) => {+ const src = path.join(dir, name);+ ffmpeg(["-y", "-f", "lavfi", "-i", "testsrc=size=320x240:rate=24:duration=1", "-f", "lavfi", "-i", "sine=duration=1",+ "-c:v", "libx264", "-pix_fmt", "yuv420p", ...colour, "-c:a", "aac", "-shortest", src]);+ return src;+ };++ test("a browser without H.264 gets VP9 picture and Opus sound in WebM", async () => {+ const src = phoneClip("phone.mp4");+ const out = path.join(dir, "phone.mp4.compat.webm");+ await makeCompatProxy(handle(out), src, out, { cover: "video", maxHeight: 180, crf: 20 });++ expect(codecsOf(out)).toEqual(["opus,audio", "vp9,video"]);+ expect(ffprobe(["-show_entries", "format=format_name", "-of", "csv=p=0", out])).toContain("webm");+ expect(Number(ffprobe(["-select_streams", "v", "-show_entries", "stream=height", "-of", "csv=p=0", out]))).toBe(180);+ }, 60_000);++ test("a Rec.709 picture is copied when only the sound needs Opus", async () => {+ const src = phoneClip("rec709.mp4", ["-colorspace", "bt709", "-color_range", "tv"]);+ const out = path.join(dir, "rec709.mp4.compat.mp4");+ await makeCompatProxy(handle(out), src, out, { cover: "audio", maxHeight: 1080, crf: 20 });++ expect(codecsOf(out)).toEqual(["h264,video", "opus,audio"]);+ const packets = (file: string) => ffprobe(["-select_streams", "v", "-show_entries", "packet=size", "-of", "csv=p=0", file]);+ expect(packets(out)).toBe(packets(src));+ }, 60_000);++ test("a picture outside Rec.709 limited is carried into it", async () => {+ const src = phoneClip("fullrange.mp4", ["-colorspace", "smpte170m", "-color_range", "pc"]);+ const out = path.join(dir, "fullrange.mp4.compat.mp4");+ await makeCompatProxy(handle(out), src, out, { cover: "audio", maxHeight: 1080, crf: 20 });++ expect(codecsOf(out)).toEqual(["h264,video", "opus,audio"]);+ expect(ffprobe(["-select_streams", "v", "-show_entries", "stream=color_space,color_range", "-of", "csv=p=0", out]).trim()).toBe("tv,bt709");+ }, 60_000);++ test("an audio file gets Opus sound alone", async () => {+ const src = path.join(dir, "voice.m4a");+ ffmpeg(["-y", "-f", "lavfi", "-i", "sine=duration=1", "-c:a", "aac", src]);+ const out = path.join(dir, "voice.m4a.compat.mp4");+ await makeCompatProxy(handle(out), src, out, { cover: "audio", maxHeight: 1080, crf: 20 });++ expect(codecsOf(out)).toEqual(["opus,audio"]);+ }, 60_000);+});diff --git a/site/src/cut/server/proxy.ts b/site/src/cut/server/proxy.tsindex 0c0337d6..4c89ba4e 100644--- a/site/src/cut/server/proxy.ts+++ b/site/src/cut/server/proxy.ts@@ -2,7 +2,9 @@ import { spawn } from "node:child_process";import { stat } from "node:fs/promises";import { runFfmpeg, videoEncoder, vtQuality, type RenderHandle } from "./exportPipeline";import { assertGraphSafe } from "./filterGraph";+import { streamCodecs } from "./convert";import { videoDimensions } from "./util";+import type { CompatCover } from "../lib/proxyNames";/*** The preview proxy of a ProRes master, or of a heavy video a cloud project@@ -194,3 +196,106 @@ export async function makeProxy(const [info, outDims] = await Promise.all([stat(out), videoDimensions(out)]);return { sizeBytes: info.size, ...(outDims ? { width: outDims.width, height: outDims.height } : {}) };}++/** VP9's crf on its own 0–63 scale, from the x265 crf every proxy is+ * configured with: 20 maps to 32. */+export function vp9Crf(crf: number) {+ return Math.round(Math.max(15, Math.min(50, crf * 1.6)));+}++/** The compat proxy's sound: Opus, which every browser decodes. */+const OPUS_KBPS = 128;++/** A keyframe every second of the master's time, counted from the last+ * forced one, as the HEVC proxy places them. */+const KEY_EVERY_SECOND = "expr:if(isnan(prev_forced_t),1,gte(t,prev_forced_t+1))";++export interface CompatArgs {+ cover: CompatCover;+ /** The proxy's largest height; a taller master is scaled down. */+ maxHeight: number;+ /** x265 crf, mapped to VP9's scale or handed to x264. */+ crf: number;+ /** A light proxy's bitrate ceiling, kbit/s. */+ maxKbps?: number;+}++/** The ffmpeg argument list for one compat proxy (lib/proxyNames.ts). Its+ * picture keeps the proxy contract: Rec.709 limited code values on the+ * master's timestamps, so every reader treats it as any proxy. `source` is+ * null for a file with no picture; `h264` names this machine's encoder. */+export function compatFfmpegArgs(+ src: string,+ out: string,+ opts: CompatArgs & { source: ProxySource | null; hasAudio: boolean; h264: string }+): string[] {+ const { source } = opts;+ const cap = Math.max(2, Math.round(opts.maxHeight));+ const kbps = opts.maxKbps && opts.maxKbps > 0 ? Math.round(opts.maxKbps) : 0;+ const video: string[] = [];+ if (!source) {+ video.push("-vn");+ } else {+ video.push("-map", "0:v:0");+ // A sound-only cover keeps the master's size unless it is a light one.+ const shrink = source.height > cap && (opts.cover === "video" || !!kbps);+ // A Rec.709 limited picture already is what a proxy holds, so a+ // sound-only cover copies it packet for packet.+ const copy = opts.cover === "audio" && source.matrix === "bt709" && !source.fullRange && !shrink && !kbps;+ if (copy) {+ video.push("-c:v", "copy");+ } else {+ const size = shrink ? `:w=-2:h=${cap - (cap % 2)}` : "";+ const graph = [+ `scale=in_color_matrix=${SWS_MATRIX[source.matrix]}:in_range=${source.fullRange ? "pc" : "tv"}:out_range=pc${size}`,+ "format=gbrp16le",+ "scale=out_color_matrix=bt709:out_range=tv",+ "format=yuv420p",+ "setparams=color_primaries=bt709:color_trc=iec61966-2-1:colorspace=bt709:range=tv",+ ].join(",");+ video.push("-vf", assertGraphSafe(graph), "-color_primaries", "bt709", "-color_trc", "iec61966-2-1", "-colorspace", "bt709", "-color_range", "tv");+ if (opts.cover === "video") {+ video.push(+ "-c:v", "libvpx-vp9", "-crf", String(vp9Crf(opts.crf)), "-b:v", kbps ? `${kbps}k` : "0",+ "-deadline", "good", "-cpu-used", "4", "-row-mt", "1"+ );+ } else if (opts.h264 === "libx264") {+ video.push("-c:v", "libx264", "-preset", "medium", "-crf", String(opts.crf), ...(kbps ? ["-maxrate", `${kbps}k`, "-bufsize", `${kbps}k`] : []));+ } else {+ video.push("-c:v", "h264_videotoolbox", ...(kbps ? ["-b:v", `${kbps}k`] : ["-q:v", String(vtQuality(opts.crf))]), "-allow_sw", "1");+ }+ video.push("-force_key_frames", KEY_EVERY_SECOND);+ }+ }+ const audio = opts.hasAudio ? ["-map", "0:a:0", "-c:a", "libopus", "-b:a", `${OPUS_KBPS}k`] : ["-an"];+ return [+ "-y",+ "-copyts",+ "-i", src,+ ...video,+ ...audio,+ ...(opts.cover === "video" ? [] : ["-movflags", "+faststart"]),+ out,+ ];+}++/** Build the compat proxy of `src` at `out`: VP9 and Opus in WebM, or the+ * master's picture with Opus sound in MP4 (lib/proxyNames.ts). */+export async function makeCompatProxy(+ handle: RenderHandle,+ src: string,+ out: string,+ opts: CompatArgs & { onProgress?: (seconds: number) => void }+): Promise<ProxyOutcome> {+ const codecs = await streamCodecs(src);+ if (opts.cover === "video" && !codecs.video) throw new Error("That file carries no video.");+ const [source, dims, h264] = await Promise.all([+ codecs.video ? probeProxySource(src) : Promise.resolve(null),+ codecs.video ? videoDimensions(src) : Promise.resolve(null),+ videoEncoder("h264"),+ ]);+ if (source && dims) source.height = dims.height;+ await runFfmpeg(handle, compatFfmpegArgs(src, out, { ...opts, source, hasAudio: !!codecs.audio, h264 }), opts.onProgress);+ const [info, outDims] = await Promise.all([stat(out), source ? videoDimensions(out) : Promise.resolve(null)]);+ return { sizeBytes: info.size, ...(outDims ? { width: outDims.width, height: outDims.height } : {}) };+}diff --git a/site/src/cut/worker/convertJob.ts b/site/src/cut/worker/convertJob.tsindex 9d10c58a..3603ec03 100644--- a/site/src/cut/worker/convertJob.ts+++ b/site/src/cut/worker/convertJob.ts@@ -4,6 +4,7 @@ import { mkdtemp, rm } from "node:fs/promises";import os from "node:os";import path from "node:path";import type { RenderHandle } from "../server/exportPipeline";+import { convertModeIn } from "../lib/convertMode";import { convertToMp4, mp4NameFor } from "../server/convert";import { probeDuration } from "../server/frames";import { dedupeName } from "../server/cloud/util";@@ -38,7 +39,8 @@ export async function runConvertJob(if ((job.spec as { target?: string } | null)?.target === "library") {return runLibraryUploadJob(job, handle, isCanceled);}- const { file, maxHeight } = (job.spec ?? {}) as { file?: string; maxHeight?: number };+ const { file, maxHeight, mode: asked } = (job.spec ?? {}) as { file?: string; maxHeight?: number; mode?: unknown };+ const mode = convertModeIn(asked) ?? "fit";if (!file) throw new Error("Convert job has no file.");const projectId = job.projectId;if (!projectId) throw new Error("Convert job has no project.");@@ -61,6 +63,7 @@ export async function runConvertJob(const total = await probeDuration(src);const outcome = await convertToMp4(handle, src, out, {maxHeight,+ mode,onProgress: (seconds) => {if (total > 0) handle.progress = Math.min(0.99, seconds / total);},diff --git a/site/src/cut/worker/libraryProxyJob.test.ts b/site/src/cut/worker/libraryProxyJob.test.tsindex 75e7c47c..6ee0c15b 100644--- a/site/src/cut/worker/libraryProxyJob.test.ts+++ b/site/src/cut/worker/libraryProxyJob.test.ts@@ -6,7 +6,7 @@ import { runLibraryProxyJob, type LibraryProxyIO } from "./libraryProxyJob";type Row = { id: string; userId: string; mediaObjectId: string; folderId: string | null; deletedAt: Date | null; meta: unknown; createdAt: Date };-function harness(taken: string[] = []) {+function harness(taken: string[] = [], compat?: string) {const row: Row = {id: "asset", userId: "owner", mediaObjectId: "master", folderId: "folder", deletedAt: null,meta: { name: "IMG_5388.MOV", type: "video", duration: 21.8, width: 1080, height: 1920, title: "Beach" },@@ -17,7 +17,9 @@ function harness(taken: string[] = []) {const takenRead = mock(async (_args: unknown) => { void _args; return taken.map((fileName) => ({ fileName })); });const update = mock(async ({ data }: { data: { meta: unknown } }) => ({ ...row, meta: data.meta }));const liveRead = mock(async (_args: unknown) => { void _args; return row as Row | null; });- const tx = { cutLibraryAsset: { findFirst: liveRead, update } };+ const oldRead = mock(async (_args: unknown) => { void _args; return { id: "old-proxy", r2Key: "cut/owner/library/IMG_5388.proxy.mp4" } as { id: string; r2Key: string } | null; });+ const oldDelete = mock(async (_args: unknown) => { void _args; });+ const tx = { cutLibraryAsset: { findFirst: liveRead, update }, cutMediaObject: { findFirst: oldRead, delete: oldDelete } };const io = {db: {cutLibraryAsset: { findFirst: assetRead },@@ -31,15 +33,20 @@ function harness(taken: string[] = []) {await writeFile(out, "proxy");return { sizeBytes: 5, width: 608, height: 1080 };}),+ makeCompat: mock(async (_handle: unknown, _src: string, out: string, opts: { cover: string }) => {+ void opts;+ await writeFile(out, "compat");+ return { sizeBytes: 6 };+ }),duration: mock(async () => 21.8),settings: mock(async () => ({ proxyMaxHeight: 1080, proxyCrf: 20 })),transact: async <T>(run: (tx: Prisma.TransactionClient) => Promise<T>) => run(tx as unknown as Prisma.TransactionClient),register: mock(async (_tx: Prisma.TransactionClient, object: { fileName: string }) => object.fileName),} satisfies LibraryProxyIO;- const job = { id: "library-proxy-asset", userId: "owner", projectId: null, kind: "proxy", outName: null, spec: { target: "library", assetId: "asset" } };+ const job = { id: "library-proxy-asset", userId: "owner", projectId: null, kind: "proxy", outName: null, spec: { target: "library", assetId: "asset", ...(compat ? { compat } : {}) } };const handle = { tmpDir: "", outPath: "", progress: 0, log: [] as string[] };const run = (canceled = () => false) => runLibraryProxyJob(job, handle, canceled, io);- return { run, io, row, update, liveRead, takenRead, handle };+ return { run, io, row, update, liveRead, takenRead, handle, oldDelete };}describe("library proxy worker", () => {@@ -72,6 +79,24 @@ describe("library proxy worker", () => {expect(h.io.download).not.toHaveBeenCalled();});+ test("a compat ask replaces a copy this browser cannot play and drops the old object", async () => {+ const h = harness([], "video");+ h.row.meta = { ...(h.row.meta as object), proxyFile: "IMG_5388.proxy.mp4" };+ expect((await h.run()).proxyFile).toBe("IMG_5388.MOV.compat.webm");+ expect(h.io.make).not.toHaveBeenCalled();+ expect(h.io.makeCompat.mock.calls[0][3]).toMatchObject({ cover: "video", maxHeight: 1080, crf: 20 });+ expect(h.io.register.mock.calls[0][1]).toMatchObject({ fileName: "IMG_5388.MOV.compat.webm", mime: "video/webm" });+ expect(h.oldDelete.mock.calls[0][0]).toEqual({ where: { id: "old-proxy" } });+ expect(h.io.remove.mock.calls[0][0]).toEqual(["cut/owner/library/IMG_5388.proxy.mp4"]);+ });++ test("a compat ask the copy already covers returns without building", async () => {+ const h = harness([], "audio");+ h.row.meta = { ...(h.row.meta as object), proxyFile: "IMG_5388.MOV.compat.webm" };+ expect((await h.run()).proxyFile).toBe("IMG_5388.MOV.compat.webm");+ expect(h.io.download).not.toHaveBeenCalled();+ });+test("an asset deleted while the copy was built takes the copy with it", async () => {const h = harness();h.liveRead.mockResolvedValue(null);diff --git a/site/src/cut/worker/libraryProxyJob.ts b/site/src/cut/worker/libraryProxyJob.tsindex 038ec0dd..cf594ebc 100644--- a/site/src/cut/worker/libraryProxyJob.ts+++ b/site/src/cut/worker/libraryProxyJob.ts@@ -4,7 +4,8 @@ import path from "node:path";import type { Prisma } from "@/generated/prisma/client";import { getGlobalSetting } from "@/lib/config/effective";import type { RenderHandle } from "../server/exportPipeline";-import { makeProxy, proxyNameFor } from "../server/proxy";+import { makeCompatProxy, makeProxy, proxyNameFor } from "../server/proxy";+import { compatCoverIn, compatCovers, compatProxyNameFor } from "../lib/proxyNames";import { probeDuration } from "../server/frames";import { assetView } from "../server/cloud/library";import { dedupeName } from "../server/cloud/util";@@ -19,6 +20,7 @@ export interface LibraryProxyIO {upload: typeof uploadFile;remove: typeof deleteObjects;make: typeof makeProxy;+ makeCompat: typeof makeCompatProxy;duration: typeof probeDuration;settings: () => Promise<{ proxyMaxHeight: number; proxyCrf: number }>;transact: <T>(run: (tx: Prisma.TransactionClient) => Promise<T>) => Promise<T>;@@ -31,6 +33,7 @@ const defaultIO: LibraryProxyIO = {upload: uploadFile,remove: deleteObjects,make: makeProxy,+ makeCompat: makeCompatProxy,duration: probeDuration,settings: () => getGlobalSetting("cutColor"),transact: storageTransaction,@@ -52,7 +55,8 @@ export async function runLibraryProxyJob(isCanceled: () => boolean,io: LibraryProxyIO = defaultIO): Promise<LibraryAsset> {- const { assetId } = (job.spec ?? {}) as { assetId?: string };+ const { assetId, compat: asked } = (job.spec ?? {}) as { assetId?: string; compat?: unknown };+ const compat = compatCoverIn(asked);if (!assetId) throw new Error("Library proxy job has no asset.");const row = await io.db.cutLibraryAsset.findFirst({ where: { id: assetId, userId: job.userId, deletedAt: null } });if (!row) throw new Error(DELETED);@@ -60,7 +64,8 @@ export async function runLibraryProxyJob(if (!media) throw new Error("Library media is missing.");// A replay, or a second ask after the copy landed, returns what is there.- if ((row.meta as { proxyFile?: string } | null)?.proxyFile) return assetView(row, media);+ const had = (row.meta as { proxyFile?: string } | null)?.proxyFile;+ if (had && (!compat || compatCovers(had, compat))) return assetView(row, media);const tmp = await mkdtemp(path.join(os.tmpdir(), "cut-library-proxy-"));handle.tmpDir = tmp;@@ -77,34 +82,51 @@ export async function runLibraryProxyJob(where: { userId: job.userId, projectId: null, kind: { in: ["library", "proxy"] } },select: { fileName: true },});- const fileName = dedupeName(proxyNameFor(media.fileName), new Set(rows.map((r) => r.fileName)));+ const name = compat ? compatProxyNameFor(media.fileName, compat, false) : proxyNameFor(media.fileName);+ const fileName = dedupeName(name, new Set(rows.map((r) => r.fileName)));const out = path.join(tmp, fileName);handle.outPath = out;const [total, settings] = await Promise.all([io.duration(src), io.settings()]);- await io.make(handle, src, out, {+ const opts = {maxHeight: settings.proxyMaxHeight,crf: settings.proxyCrf,- onProgress: (seconds) => {+ onProgress: (seconds: number) => {if (total > 0) handle.progress = Math.min(0.99, seconds / total);},- });+ };+ await (compat ? io.makeCompat(handle, src, out, { ...opts, cover: compat }) : io.make(handle, src, out, opts));if (isCanceled()) throw new Error("Proxy canceled.");key = libraryKey(job.userId, fileName);- const bytes = await io.upload(key, out, "video/mp4");+ const mime = compat === "video" ? "video/webm" : "video/mp4";+ const bytes = await io.upload(key, out, mime);+ let replaced: string | undefined;const stored = key;const updated = await io.transact(async (tx) => {// The asset may have been deleted while the copy was built.const live = await tx.cutLibraryAsset.findFirst({ where: { id: assetId, userId: job.userId, deletedAt: null } });if (!live) throw new Error(DELETED);await io.register(tx, {- userId: job.userId, projectId: null, r2Key: stored, fileName, mime: "video/mp4", bytes, kind: "proxy",+ userId: job.userId, projectId: null, r2Key: stored, fileName, mime, bytes, kind: "proxy",});+ // A compat copy takes the place of a copy this browser cannot play;+ // the old one's object leaves with its name.+ const before = (live.meta as { proxyFile?: string } | null)?.proxyFile;+ if (before) {+ const old = await tx.cutMediaObject.findFirst({ where: { userId: job.userId, projectId: null, kind: "proxy", fileName: before } });+ if (old) {+ await tx.cutMediaObject.delete({ where: { id: old.id } });+ replaced = old.r2Key;+ }+ }const meta = { ...((live.meta ?? {}) as Record<string, unknown>), proxyFile: fileName };return tx.cutLibraryAsset.update({ where: { id: assetId }, data: { meta: meta as Prisma.InputJsonValue } });});published = true;+ if (replaced) {+ await io.remove([replaced]).catch(() => {});+ }return assetView(updated, media);} finally {try {diff --git a/site/src/cut/worker/libraryUploadJob.test.ts b/site/src/cut/worker/libraryUploadJob.test.tsindex 16de38ef..0c34548c 100644--- a/site/src/cut/worker/libraryUploadJob.test.ts+++ b/site/src/cut/worker/libraryUploadJob.test.ts@@ -23,7 +23,7 @@ function harness() {upload: mock(async (_key: string, _file: string, _mime: string) => { void _file; void _mime; return 5; }),remove: mock(async (_keys: string[]) => { void _keys; }),convert: mock(async () => ({ transcodedVideo: true, transcodedAudio: false, width: 64, height: 64, unchanged: false })),- codecs: mock(async () => ({ video: "hevc", audio: "aac", videoIndex: 0, audioIndex: 1, remux: false, videoNeedsEncoding: true })),+ codecs: mock(async () => ({ video: "hevc", audio: "aac", videoIndex: 0, audioIndex: 1, remux: false, videoNeedsEncoding: true, mp4: true })),duration: mock(async () => 2),transact: async <T>(run: (tx: Prisma.TransactionClient) => Promise<T>) => run(tx as unknown as Prisma.TransactionClient),register,diff --git a/site/src/cut/worker/libraryUploadJob.ts b/site/src/cut/worker/libraryUploadJob.tsindex c8d7cbbc..c480b69d 100644--- a/site/src/cut/worker/libraryUploadJob.ts+++ b/site/src/cut/worker/libraryUploadJob.ts@@ -56,6 +56,7 @@ export async function runLibraryUploadJob(handle.outPath = out;const converted = await io.convert(handle, src, out, {maxHeight: input.maxHeight,+ mode: input.mode,onProgress: (seconds) => { handle.progress = Math.min(0.95, seconds / duration); },});checkCanceled();diff --git a/site/src/cut/worker/proxyJob.test.ts b/site/src/cut/worker/proxyJob.test.tsindex 5f5b89d7..5bb83be7 100644--- a/site/src/cut/worker/proxyJob.test.ts+++ b/site/src/cut/worker/proxyJob.test.ts@@ -20,6 +20,11 @@ function harness(taken: string[] = [], spec: Record<string, unknown> = { file: "await writeFile(out, "proxy");return { sizeBytes: 5, width: 64, height: 64 };}),+ makeCompat: mock(async (_handle: unknown, _src: string, out: string, opts: { cover: string; maxHeight: number; crf: number }) => {+ void opts;+ await writeFile(out, "compat");+ return { sizeBytes: 6, width: 64, height: 36 };+ }),duration: mock(async () => 2),settings: mock(async () => ({ proxyMaxHeight: 1080, proxyCrf: 20, lightKbps: 6000 })),} satisfies ProxyJobIO;@@ -40,6 +45,18 @@ describe("proxy worker", () => {expect((pro.io.make.mock.calls[0][3] as { maxKbps?: number }).maxKbps).toBeUndefined();});+ test("a compat job builds the cover it names and registers it by its container", async () => {+ const h = harness([], { file: "clip.mp4", compat: "video" });+ expect((await h.run()).fileName).toBe("clip.mp4.compat.webm");+ expect(h.io.make).not.toHaveBeenCalled();+ expect(h.io.makeCompat.mock.calls[0][3]).toMatchObject({ cover: "video", maxHeight: 1080, crf: 20 });+ expect(h.io.register.mock.calls[0][0]).toMatchObject({ fileName: "clip.mp4.compat.webm", kind: "proxy", mime: "video/webm" });+ // A heavy video streamed from the cloud gets the light one.+ const light = harness([], { file: "clip.mp4", compat: "audio", maxHeight: 1080 });+ expect((await light.run()).fileName).toBe("clip.mp4.compat.light.mp4");+ expect(light.io.makeCompat.mock.calls[0][3]).toMatchObject({ cover: "audio", maxKbps: 6000 });+ });+test("a job that asks for a smaller copy gets it, never one past the account's size", async () => {const small = harness([], { file: "clip.mov", maxHeight: 720 });await small.run();diff --git a/site/src/cut/worker/proxyJob.ts b/site/src/cut/worker/proxyJob.tsindex 1c9229be..0f546a8d 100644--- a/site/src/cut/worker/proxyJob.ts+++ b/site/src/cut/worker/proxyJob.ts@@ -3,8 +3,8 @@ import os from "node:os";import path from "node:path";import { getGlobalSetting } from "@/lib/config/effective";import type { RenderHandle } from "../server/exportPipeline";-import { makeProxy, proxyNameFor } from "../server/proxy";-import { lightProxyNameFor, PROXY_SOURCE_FAULT } from "../lib/proxyNames";+import { makeCompatProxy, makeProxy, proxyNameFor } from "../server/proxy";+import { compatCoverIn, compatProxyNameFor, lightProxyNameFor, PROXY_SOURCE_FAULT } from "../lib/proxyNames";import { probeDuration } from "../server/frames";import { dedupeName } from "../server/cloud/util";import { prisma, registerObject, unregisterObjects, type ClaimedJob } from "./db";@@ -31,6 +31,7 @@ export interface ProxyJobIO {register: typeof registerObject;unregister: typeof unregisterObjects;make: typeof makeProxy;+ makeCompat: typeof makeCompatProxy;duration: typeof probeDuration;settings: () => Promise<{ proxyMaxHeight: number; proxyCrf: number; lightKbps: number }>;}@@ -43,6 +44,7 @@ const defaultIO: ProxyJobIO = {register: registerObject,unregister: unregisterObjects,make: makeProxy,+ makeCompat: makeCompatProxy,duration: probeDuration,settings: async () => {const [color, proxy] = await Promise.all([getGlobalSetting("cutColor"), getGlobalSetting("cutProxy")]);@@ -67,7 +69,8 @@ export async function runProxyJob(if ((job.spec as { target?: string } | null)?.target === "library") {return runLibraryProxyJob(job, handle, isCanceled);}- const { file, maxHeight } = (job.spec ?? {}) as { file?: string; maxHeight?: number };+ const { file, maxHeight, compat: asked } = (job.spec ?? {}) as { file?: string; maxHeight?: number; compat?: unknown };+ const compat = compatCoverIn(asked);if (!file) throw new Error("Proxy job has no file.");const projectId = job.projectId;if (!projectId) throw new Error("Proxy job has no project.");@@ -86,21 +89,27 @@ export async function runProxyJob(// A job that names a height is a heavy video's: its proxy is the light// copy, capped in bitrate and named apart from a full-quality one.const light = !!(maxHeight && maxHeight > 0);- const fileName = dedupeName(light ? lightProxyNameFor(file) : proxyNameFor(file), new Set(rows.map((r) => r.fileName)));+ let name = light ? lightProxyNameFor(file) : proxyNameFor(file);+ if (compat) {+ name = compatProxyNameFor(file, compat, light);+ }+ const fileName = dedupeName(name, new Set(rows.map((r) => r.fileName)));const out = path.join(tmp, fileName);handle.outPath = out;const [total, settings] = await Promise.all([io.duration(src), io.settings()]);- const outcome = await io.make(handle, src, out, {+ const opts = {// A heavy video's proxy asks for a smaller copy; none is larger than// the account allows.maxHeight: maxHeight && maxHeight > 0 ? Math.min(settings.proxyMaxHeight, maxHeight) : settings.proxyMaxHeight,crf: settings.proxyCrf,...(light ? { maxKbps: settings.lightKbps } : {}),- onProgress: (seconds) => {+ onProgress: (seconds: number) => {if (total > 0) handle.progress = Math.min(0.99, seconds / total);},- }).catch((e: unknown) => {+ };+ const made = compat ? io.makeCompat(handle, src, out, { ...opts, cover: compat }) : io.make(handle, src, out, opts);+ const outcome = await made.catch((e: unknown) => {// ffmpeg ran to its own end and refused the master: the row says the// file is at fault, so the route waits before trying it again.throw !isCanceled() && refusedInput(handle.proc) ? new InputFault(PROXY_SOURCE_FAULT, e) : e;@@ -109,13 +118,14 @@ export async function runProxyJob(const key = mediaKey(job.userId, projectId, fileName);try {- const bytes = await io.upload(key, out, "video/mp4");+ const mime = compat === "video" ? "video/webm" : "video/mp4";+ const bytes = await io.upload(key, out, mime);await io.register({userId: job.userId,projectId,r2Key: key,fileName,- mime: "video/mp4",+ mime,bytes,kind: "proxy",});