Skip to content

Code-first video & image, rendered natively on WebGPU.

feat(renderer-service): stitch inputs from presigned URLs

okanasle9b7e6238463
diff --git a/apps/renderer-service/src/jobs/jobs.test.ts b/apps/renderer-service/src/jobs/jobs.test.ts
index 291bbfc4..1954d61e 100644
--- a/apps/renderer-service/src/jobs/jobs.test.ts
+++ b/apps/renderer-service/src/jobs/jobs.test.ts
@@ -97,10 +97,59 @@ describe("validateJob", () => {
{ render: { kind: "frames" } },
"JOB_INVALID",
],
+ [
+ "stitch keys without s3 storage",
+ { render: { kind: "stitch", stitch: { chunks: ["r/chunk-0.mp4"] } } },
+ "JOB_INVALID",
+ ],
+ [
+ "plain-http stitch URLs",
+ {
+ render: {
+ kind: "stitch",
+ stitch: { chunks: ["http://r2.example.com/chunk-0.mp4?sig=x"] },
+ },
+ },
+ "JOB_INVALID",
+ ],
+ [
+ "a range on a non-video kind",
+ { render: { kind: "audio", range: { from: 0, to: 10 } } },
+ "JOB_INVALID",
+ ],
])("rejects %s", (_, overrides, code) => {
expect(codeOf(() => validateJob(testJob(overrides), config))).toBe(code);
});
+ it("takes split-render jobs; presigned stitch inputs need no storage keys", () => {
+ const chunk = validateJob(
+ testJob({ render: { range: { from: 0, to: 60 }, audio: false } }),
+ config,
+ );
+ expect(chunk.input.render).toMatchObject({
+ kind: "video",
+ range: { from: 0, to: 60 },
+ audio: false,
+ });
+ const stitch = validateJob(
+ testJob({
+ render: {
+ kind: "stitch",
+ stitch: {
+ chunks: [
+ "https://acct.r2.cloudflarestorage.com/b/chunk-000.mp4?X-Amz-Signature=a",
+ "https://acct.r2.cloudflarestorage.com/b/chunk-001.mp4?X-Amz-Signature=b",
+ ],
+ audio:
+ "https://acct.r2.cloudflarestorage.com/b/audio.m4a?X-Amz-Signature=c",
+ },
+ },
+ }),
+ config,
+ );
+ expect(stitch.input.render.stitch?.chunks).toHaveLength(2);
+ });
+
it("enforces the bucket allowlist", () => {
const s3 = testConfig({
STORAGE_ENDPOINT: "https://s3.example.com",
diff --git a/apps/renderer-service/src/jobs/schema.ts b/apps/renderer-service/src/jobs/schema.ts
index 9fe1c908..01ec78ad 100644
--- a/apps/renderer-service/src/jobs/schema.ts
+++ b/apps/renderer-service/src/jobs/schema.ts
@@ -72,6 +72,15 @@ const storageKey = z.string().min(1).max(1024).refine(isSafeRelativePath, {
message: 'keys must be relative and must not contain ".."',
});
+/**
+ * A stitch input: an https URL (presigned GET, so the worker needs no
+ * storage keys) or an object key in the output bucket.
+ */
+const stitchInput = z.union([
+ z.url({ protocol: /^https$/ }).max(8192),
+ storageKey,
+]);
+
export const renderSchema = z.object({
kind: z.enum(RENDER_KINDS).default("video"),
codec: z.string().max(32).optional(),
@@ -96,12 +105,12 @@ export const renderSchema = z.object({
minChunkFrames: z.number().int().min(1).optional(),
})
.optional(),
- /** kind=stitch: chunk files (and the audio file) in the output bucket, joined in order. */
+ /** kind=stitch: chunk files (and the audio file), as presigned URLs or keys in the output bucket, joined in order. */
stitch: z
.object({
bucket: z.string().min(1).max(255).optional(),
- chunks: z.array(storageKey).min(1).max(256),
- audio: storageKey.optional(),
+ chunks: z.array(stitchInput).min(1).max(256),
+ audio: stitchInput.optional(),
})
.optional(),
grid: z
diff --git a/apps/renderer-service/src/jobs/validate.ts b/apps/renderer-service/src/jobs/validate.ts
index af3d0f5c..7137da44 100644
--- a/apps/renderer-service/src/jobs/validate.ts
+++ b/apps/renderer-service/src/jobs/validate.ts
@@ -138,10 +138,12 @@ function validateOutput(input: JobInput, config: RendererConfig): void {
}
}
-/** A stitch job reads its chunks with the worker's own storage credentials. */
+/** Stitch inputs given as keys are read with the worker's own storage credentials. */
function validateStitchInputs(input: JobInput, config: RendererConfig): void {
const stitch = input.render.stitch;
if (input.render.kind !== "stitch" || !stitch) return;
+ const inputs = [...stitch.chunks, ...(stitch.audio ? [stitch.audio] : [])];
+ if (inputs.every(isUrl)) return;
if (config.storage.driver !== "s3") {
throw new RenderError(
"JOB_INVALID",
@@ -157,6 +159,10 @@ function validateStitchInputs(input: JobInput, config: RendererConfig): void {
}
}
+export function isUrl(input: string): boolean {
+ return input.startsWith("https://");
+}
+
function globMatch(pattern: string, value: string): boolean {
const re = new RegExp(
`^${pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*")}$`,
diff --git a/apps/renderer-service/src/output/sinks.ts b/apps/renderer-service/src/output/sinks.ts
index 41ff0bb6..2398b5bb 100644
--- a/apps/renderer-service/src/output/sinks.ts
+++ b/apps/renderer-service/src/output/sinks.ts
@@ -1,8 +1,9 @@
import { createReadStream, createWriteStream } from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
-import type { Readable } from "node:stream";
+import { Readable } from "node:stream";
import { pipeline } from "node:stream/promises";
+import type { ReadableStream as WebReadableStream } from "node:stream/web";
import {
GetObjectCommand,
HeadObjectCommand,
@@ -124,6 +125,30 @@ export async function downloadS3(
}
}
+/** Downloads a presigned URL to `file` (a stitch job's chunks). */
+export async function downloadUrl(
+ url: string,
+ file: string,
+ signal: AbortSignal,
+): Promise<void> {
+ await withRetries(signal, async () => {
+ const res = await fetch(url, { signal });
+ if (res.status === 404 || res.status === 403)
+ throw new RenderError(
+ "ASSET_NOT_FOUND",
+ `Stitch input ${redact(url.split("?")[0])} is missing or its URL expired (HTTP ${res.status}).`,
+ { phase: "render" },
+ );
+ if (!res.ok || !res.body)
+ throw new Error(`Stitch input download failed: HTTP ${res.status}`);
+ await pipeline(
+ Readable.fromWeb(res.body as WebReadableStream),
+ createWriteStream(file),
+ { signal },
+ );
+ });
+}
+
export function publicUrl(
config: RendererConfig,
key: string,
diff --git a/apps/renderer-service/src/render/render-step.ts b/apps/renderer-service/src/render/render-step.ts
index 8fc322fb..461e6485 100644
--- a/apps/renderer-service/src/render/render-step.ts
+++ b/apps/renderer-service/src/render/render-step.ts
@@ -9,8 +9,9 @@ import type {
} from "../jobs/pipeline.js";
import type { JobInput, RenderSettings } from "../jobs/schema.js";
import type { JobProgress } from "../jobs/store.js";
+import { isUrl } from "../jobs/validate.js";
import { rendererLogger } from "../logger.js";
-import { downloadS3 } from "../output/sinks.js";
+import { downloadS3, downloadUrl } from "../output/sinks.js";
import { gpuStatus } from "../worker/gpu.js";
import { workerHealth } from "../worker/health.js";
import {
@@ -458,29 +459,33 @@ async function stitchOutput(
engine: BoundEngine,
): Promise<RenderedFile> {
const stitch = settings.stitch;
- const bucket = stitch?.bucket ?? ctx.config.storage.bucket;
- if (!stitch || !bucket)
- throw new RenderError(
- "JOB_INVALID",
- "render.stitch needs chunks and a bucket.",
- );
+ if (!stitch)
+ throw new RenderError("JOB_INVALID", "render.stitch is missing.");
+ const bucket = stitch.bucket ?? ctx.config.storage.bucket;
const stitchChunks = requireEngine(engine, "stitchChunks", "stitch jobs");
const inDir = path.join(ctx.jobDir, "in");
await fs.mkdir(inDir, { recursive: true });
- const keys = [...stitch.chunks, ...(stitch.audio ? [stitch.audio] : [])];
+ const inputs = [...stitch.chunks, ...(stitch.audio ? [stitch.audio] : [])];
let downloaded = 0;
- const local = await mapLimit(keys, 4, async (key, i) => {
- const file = path.join(inDir, `${i}-${path.basename(key)}`);
- await downloadS3(ctx.config, bucket, key, file, controller.signal);
+ const local = await mapLimit(inputs, 4, async (input, i) => {
+ const file = path.join(inDir, `${i}-${path.basename(inputPath(input))}`);
+ if (isUrl(input)) await downloadUrl(input, file, controller.signal);
+ else if (bucket)
+ await downloadS3(ctx.config, bucket, input, file, controller.signal);
+ else
+ throw new RenderError(
+ "JOB_INVALID",
+ "Stitch inputs given as keys need STORAGE_BUCKET or stitch.bucket.",
+ );
ctx.progress({
phase: "rendering",
frame: ++downloaded,
- total: keys.length,
+ total: inputs.length,
message: "downloading chunks",
});
return file;
});
- const ext = path.extname(stitch.chunks[0]).slice(1) || "mp4";
+ const ext = path.extname(inputPath(stitch.chunks[0])).slice(1) || "mp4";
const { contentType } = containerFor(ext === "webm" ? "vp9" : "h264");
const outputPath = path.join(outDir, `0-main.${ext}`);
ctx.progress({
@@ -502,6 +507,11 @@ async function stitchOutput(
};
}
+/** A stitch input's path: the key, or a URL's path without its signature. */
+function inputPath(input: string): string {
+ return isUrl(input) ? new URL(input).pathname : input;
+}
+
/** `Promise.all` over `items`, at most `limit` at a time, results in order. */
async function mapLimit<T, R>(
items: readonly T[],