Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

99.3 KB
Raw Blame
import { createHash, randomUUID } from "node:crypto";
import { link } from "node:fs/promises";
import { file } from "bun";
import { Hono } from "hono";
import { z } from "zod";
import { enhanceLocalRecording } from "../lib/audio-levels";
import {
	canReadStatus,
	hasMediaServerSecret,
	validateMediaServerSecret,
} from "../lib/auth";
import type {
	Job,
	RecordingErrorCode,
	VideoMetadata,
} from "../lib/job-manager";
import {
	beginRecordingProcessing,
	beginRecordingVerification,
	canAcceptNewVideoProcess,
	claimRecordingWorker,
	createJob,
	deleteJob,
	extendJobLifetime,
	forceCleanupActiveJobs,
	generateJobId,
	getActiveVideoProcessCount,
	getAllJobs,
	getJob,
	getJobProgress,
	getMaxConcurrentVideoProcesses,
	getSystemResources,
	hasCriticalMemoryPressure,
	sendWebhook,
	touchJob,
	updateJob,
} from "../lib/job-manager";
import {
	DRAINING_DETAILS,
	getIdleTimeoutSeconds,
	isDraining,
	setRequestIdleTimeout,
	trackBackgroundTask,
} from "../lib/lifecycle";
import { logReason } from "../lib/log-redaction";
import { PROCESS_TIMEOUT_MS, UPLOAD_TIMEOUT_MS } from "../lib/media-common";
import {
	getTimeoutMs as getEditRenderTimeoutMs,
	renderEditedVideo,
} from "../lib/media-edit";
import {
	canAcceptNewProbeOperation as canAcceptNewProbeProcess,
	getActiveProbeOperationCount as getActiveProbeProcessCount,
	probeVideo,
	probeVideoFile,
} from "../lib/media-probe";
import {
	fetchMedia,
	MediaTransferBudgetError,
	materializeMedia,
	releaseMaterializedMedia,
	withMediaTransfers,
} from "../lib/media-transfer";
import type {
	PendingStorageUpload,
	ResilientInputFlags,
	StorageUploadReceipt,
	StorageUploadTarget,
} from "../lib/media-video";
import {
	abortStorageUploadTarget,
	beginStorageUpload,
	downloadVideoToTemp,
	generatePreviewGif,
	generateThumbnail,
	getProcessTimeoutMs,
	LOOM_IMPORT_MAX_BYTES,
	muxMediaTracksToMp4,
	processVideo,
	repairContainer,
	uploadFileToS3,
	uploadFileToStorage,
	uploadToS3,
} from "../lib/media-video";
import { encodeAttemptsBudgetMs } from "../lib/nvenc-encoder";
import { findJobOnPeer, isPeerLookup } from "../lib/peer-jobs";
import {
	MAX_PREVERIFIED_RUN_BYTES,
	type PreverifiedRunPayload,
	readBoundedText,
	readPreverifiedRun,
	resolvePreverifiedRuns,
	signPreverifiedRun,
} from "../lib/recording-preverify";
import { RecordingTimingError } from "../lib/recording-timing";
import {
	decodePreverifiedRun,
	hashRecordingFile,
	isRetryableRecordingVerificationError,
	type PreverifiedVideoRun,
	verifyDecodeChunks,
	verifyRecording,
	verifyRemoteRecording,
	verifyRemoteRecordingBytes,
	verifyRemuxedRecording,
} from "../lib/recording-verification";
import { getRuntimeReport } from "../lib/runtime-report";
import type { TempFileHandle } from "../lib/temp-files";
import { cleanupStaleTempFiles } from "../lib/temp-files";
import {
	getActiveDirectVideoProcessCount,
	getMaxConcurrentDirectVideoProcesses,
	tryAcquireDirectVideoProcessSlot,
	type VideoProcessSlot,
} from "../lib/video-capacity";
import { validateVideoInput } from "../lib/video-input-validation";

const video = new Hono();
const PROCESSING_HEARTBEAT_MS = 60 * 1000;
const RECORDING_WORKER_INSTANCE = randomUUID();
const SEGMENTED_RECORDING_TIMEOUT_MS = 3 * PROCESS_TIMEOUT_MS + 35 * 60 * 1000;
const POST_VERIFICATION_ASSET_BUDGET_MS = 5 * 60 * 1000;
const RECORDING_VERIFICATION_RETRY_ERROR =
	"Recording verification temporarily unavailable (503)";
const MEDIA_ENGINE_ERROR_CODE = ["FF", "MPEG_ERROR"].join("");
const PROBE_ERROR_CODE = ["FF", "PROBE_ERROR"].join("");
const VIDEO_BUSY_RETRY_AFTER_SECONDS = 15;
const VIDEO_MEMORY_PRESSURE_ERROR =
	"SERVER_BUSY: container memory pressure is too high";
const MUX_FAILURE_LOG_LENGTH = 1_000;
const SEGMENT_DOWNLOAD_MAX_ATTEMPTS = 3;
const SEGMENT_DOWNLOAD_RETRY_BASE_MS = 250;
const JOB_TRANSFER_ALLOWANCE_MS = 2 * UPLOAD_TIMEOUT_MS;
const JOB_TRANSFER_BYTES_PER_SECOND = 2 * 1024 ** 2;

const probeSchema = z.object({
	videoUrl: z.string().url(),
});

const thumbnailSchema = z.object({
	videoUrl: z.string().url(),
	timestamp: z.number().optional(),
	width: z.number().max(2000).optional(),
	height: z.number().max(2000).optional(),
	quality: z.number().min(1).max(100).optional(),
});

const convertSchema = z.object({
	videoUrl: z.string().url(),
	inputExtension: z.string().optional(),
});

const storageUploadSchema = z.discriminatedUnion("type", [
	z.object({
		type: z.literal("put"),
		url: z.string().url(),
		ifNoneMatch: z.literal("*").optional(),
	}),
	z.object({
		type: z.literal("multipart"),
		videoId: z.string(),
		generation: z.string().min(1).max(200).optional(),
		attemptId: z.string().min(1).max(200).optional(),
		key: z.string().min(1),
		uploadId: z.string().min(1),
		partSize: z
			.number()
			.int()
			.min(5 * 1024 * 1024),
		signPartUrl: z.string().url(),
		completeUrl: z.string().url(),
		abortUrl: z.string().url(),
		webhookSecret: z.string().optional(),
		loomImportAttemptId: z.string().min(1).max(200).optional(),
	}),
]);

const processSchema = z.object({
	audioLevels: z.boolean().optional(),
	videoId: z.string(),
	userId: z.string(),
	videoUrl: z.string().url(),
	outputPresignedUrl: z.string().url(),
	sourcePresignedUrl: z.string().url().optional(),
	sourceUpload: storageUploadSchema.optional(),
	outputUpload: storageUploadSchema.optional(),
	existingOutput: z
		.object({
			url: z.string().url(),
			objectIdentity: z.string().regex(/^"[^"\r\n]+"$/),
			fileSize: z.number().int().positive().max(LOOM_IMPORT_MAX_BYTES),
		})
		.optional(),
	maxImportBytes: z.literal(LOOM_IMPORT_MAX_BYTES).optional(),
	thumbnailPresignedUrl: z.string().url().optional(),
	previewGifPresignedUrl: z.string().url().optional(),
	webhookUrl: z.string().url().optional(),
	webhookSecret: z.string().optional(),
	inputExtension: z.string().optional(),
	priority: z.enum(["normal", "bulk"]).optional(),
	maxWidth: z.number().max(4096).optional(),
	maxHeight: z.number().max(4096).optional(),
	crf: z.number().min(0).max(51).optional(),
	preset: z.enum(["ultrafast", "fast", "medium", "slow"]).optional(),
	remuxOnly: z.boolean().optional(),
	normalizeH264Level: z.boolean().optional(),
	importedOriginal: z.boolean().optional(),
});

export function isCompleteRecovery(
	outputDuration: number,
	inputDuration: number,
): boolean {
	return outputDuration >= inputDuration - Math.max(1, inputDuration * 0.01);
}

const editRangeSchema = z
	.object({
		start: z.number().min(0),
		end: z.number().min(0),
	})
	.refine((range) => range.end > range.start, {
		message: "Range end must be greater than start",
	});

const editSchema = z.object({
	videoId: z.string(),
	userId: z.string(),
	sourceUrl: z.string().url(),
	outputPresignedUrl: z.string().url(),
	outputVerificationUrl: z.string().url().optional(),
	thumbnailPresignedUrl: z.string().url().optional(),
	previewGifPresignedUrl: z.string().url().optional(),
	webhookUrl: z.string().url().optional(),
	webhookSecret: z.string().optional(),
	keepRanges: z.array(editRangeSchema).min(1),
});

function getInstanceId(): string {
	return process.env.HOSTNAME || `pid-${process.pid}`;
}

function logVideoEvent(event: string, data: Record<string, unknown>) {
	console.info(
		JSON.stringify({
			event,
			timestamp: new Date().toISOString(),
			instanceId: getInstanceId(),
			pid: process.pid,
			...data,
		}),
	);
}

function getVideoCapacitySnapshot() {
	const resources = getSystemResources();
	const jobs = getAllJobs();
	return {
		instanceId: getInstanceId(),
		pid: process.pid,
		activeVideoProcesses: getActiveVideoProcessCount(),
		activeDirectVideoProcesses: getActiveDirectVideoProcessCount(),
		maxConcurrentVideoProcesses: getMaxConcurrentVideoProcesses(),
		maxConcurrentDirectVideoProcesses: getMaxConcurrentDirectVideoProcesses(),
		effectiveMaxVideoProcesses: resources.effectiveMax,
		resources,
		jobCount: jobs.length,
		jobs: jobs.map((job) => ({
			jobId: job.jobId,
			videoId: job.videoId,
			phase: job.phase,
			progress: job.progress,
			updatedAt: job.updatedAt,
		})),
	};
}

function getBusyDetails(snapshot: ReturnType<typeof getVideoCapacitySnapshot>) {
	return snapshot.resources.throttleReason
		? `Throttled: ${snapshot.resources.throttleReason} (${snapshot.activeVideoProcesses}/${snapshot.resources.effectiveMax} active)`
		: `Too many concurrent video processing jobs (${snapshot.activeVideoProcesses}/${snapshot.resources.effectiveMax}), please retry later`;
}

function getBusyResponseBody(
	snapshot: ReturnType<typeof getVideoCapacitySnapshot>,
) {
	return {
		error: "Server is busy",
		code: "SERVER_BUSY",
		details: getBusyDetails(snapshot),
		...snapshot,
	};
}

function getMuxBusyResponseBody(
	snapshot: ReturnType<typeof getVideoCapacitySnapshot>,
) {
	return {
		...getBusyResponseBody(snapshot),
		error: "SERVER_BUSY",
		message: "Server is at capacity",
	};
}

const DRAINING_RESPONSE_BODY = {
	error: "Server is busy",
	code: "SERVER_BUSY",
	details: DRAINING_DETAILS,
};

function summarizeVideoInput(videoUrl: string, inputExtension?: string) {
	const extension = inputExtension?.toLowerCase() ?? null;
	try {
		const url = new URL(videoUrl);
		const pathnameExtension = url.pathname.includes(".")
			? `.${url.pathname.split(".").pop()?.toLowerCase() ?? ""}`
			: null;
		const effectiveExtension = extension ?? pathnameExtension;
		return {
			host: url.hostname,
			extension: effectiveExtension,
			streaming:
				effectiveExtension === ".m3u8" ||
				effectiveExtension === ".m3u" ||
				effectiveExtension === ".mpd",
		};
	} catch {
		return {
			host: null,
			extension,
			streaming: false,
		};
	}
}

function isBusyError(err: unknown): boolean {
	return (
		err instanceof Error &&
		(err.message.includes("Server is busy") ||
			err.message.includes("SERVER_BUSY"))
	);
}

function isTimeoutError(err: unknown): boolean {
	return err instanceof Error && err.message.includes("timed out");
}

async function cleanupTempFiles(
	files: Array<TempFileHandle | null>,
): Promise<void> {
	await Promise.all(
		files.map(async (tempFile) => {
			if (!tempFile) return;
			try {
				await tempFile.cleanup();
			} catch {}
		}),
	);
}

async function createVideoDownloadResponse(
	outputTempFile: TempFileHandle,
	tempFiles: TempFileHandle[],
	onCleanup?: () => void,
): Promise<Response> {
	const outputFile = file(outputTempFile.path);
	const outputSize = await outputFile.size;
	let cleanedUp = false;

	const cleanup = async () => {
		if (cleanedUp) return;
		cleanedUp = true;
		await cleanupTempFiles(tempFiles);
		onCleanup?.();
	};

	const reader = outputFile.stream().getReader();
	const stream = new ReadableStream<Uint8Array>({
		async pull(controller) {
			try {
				const { done, value } = await reader.read();
				if (done) {
					controller.close();
					reader.releaseLock();
					await cleanup();
				} else if (value) {
					controller.enqueue(value);
				}
			} catch (error) {
				controller.error(error);
				reader.releaseLock();
				await cleanup();
			}
		},
		async cancel() {
			await reader.cancel().catch(() => {});
			await cleanup();
		},
	});

	return new Response(stream, {
		headers: {
			"Content-Type": "video/mp4",
			"Cache-Control": "no-store",
			"Content-Length": outputSize.toString(),
		},
	});
}

async function withJobHeartbeat<T>(
	jobId: string,
	operation: () => Promise<T>,
): Promise<T> {
	const interval = setInterval(() => {
		const job = getJob(jobId);
		if (
			!job ||
			job.phase === "complete" ||
			job.phase === "error" ||
			job.phase === "cancelled"
		) {
			clearInterval(interval);
			return;
		}

		touchJob(jobId);
	}, PROCESSING_HEARTBEAT_MS);
	interval.unref?.();

	try {
		return await operation();
	} finally {
		clearInterval(interval);
	}
}

function extendJobLifetimeForWork(
	jobId: string,
	workTimeoutMs: number,
	transferBytes: number,
): void {
	extendJobLifetime(
		jobId,
		encodeAttemptsBudgetMs(workTimeoutMs) +
			JOB_TRANSFER_ALLOWANCE_MS +
			POST_VERIFICATION_ASSET_BUDGET_MS +
			Math.ceil(
				(Math.max(0, transferBytes || 0) / JOB_TRANSFER_BYTES_PER_SECOND) *
					1000,
			),
	);
}

async function withMuxMemoryGuard<T>(
	abortController: AbortController,
	operation: () => Promise<T>,
): Promise<T> {
	let memoryPressureExceeded = false;
	const interval = setInterval(() => {
		if (!hasCriticalMemoryPressure()) return;
		memoryPressureExceeded = true;
		abortController.abort();
	}, 1000);
	interval.unref?.();

	try {
		const result = await operation();
		if (memoryPressureExceeded) {
			throw new Error(VIDEO_MEMORY_PRESSURE_ERROR);
		}
		return result;
	} catch (error) {
		if (memoryPressureExceeded) {
			throw new Error(VIDEO_MEMORY_PRESSURE_ERROR);
		}
		throw error;
	} finally {
		clearInterval(interval);
	}
}

video.get("/status", async (c) => {
	if (!canReadStatus(c)) {
		return c.json({ error: "Unauthorized" }, 401);
	}
	const jobs = getAllJobs();
	const resources = getSystemResources();
	const now = Date.now();
	return c.json({
		instanceId: getInstanceId(),
		pid: process.pid,
		loomImportMaxBytes: LOOM_IMPORT_MAX_BYTES,
		loomMultipartUploads: true,
		activeVideoProcesses: getActiveVideoProcessCount(),
		maxConcurrentVideoProcesses: getMaxConcurrentVideoProcesses(),
		effectiveMaxVideoProcesses: resources.effectiveMax,
		maxConcurrentDirectVideoProcesses: getMaxConcurrentDirectVideoProcesses(),
		activeProbeProcesses: getActiveProbeProcessCount(),
		canAcceptNewVideoProcess: canAcceptNewVideoProcess(),
		canAcceptNewProbeProcess: canAcceptNewProbeProcess(),
		resources,
		activeDirectVideoProcesses: getActiveDirectVideoProcessCount(),
		jobCount: jobs.length,
		jobs: jobs.map((j) => ({
			jobId: j.jobId,
			videoId: j.videoId,
			phase: j.phase,
			progress: j.progress,
			createdAt: j.createdAt,
			updatedAt: j.updatedAt,
			ageMinutes: Math.round((now - j.createdAt) / 60000),
			stalenessMinutes: Math.round((now - j.updatedAt) / 60000),
			error: j.error,
		})),
		...(hasMediaServerSecret(c) && { runtime: await getRuntimeReport() }),
	});
});

video.post("/probe", async (c) => {
	if (!validateMediaServerSecret(c)) {
		return c.json({ error: "Unauthorized" }, 401);
	}

	const body = await c.req.json();
	const result = probeSchema.safeParse(body);

	if (!result.success) {
		return c.json(
			{
				error: "Invalid request",
				code: "INVALID_REQUEST",
				details: result.error.message,
			},
			400,
		);
	}

	if (isDraining()) {
		c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
		return c.json(DRAINING_RESPONSE_BODY, 503);
	}

	try {
		const metadata = await probeVideo(result.data.videoUrl);
		return c.json({ metadata });
	} catch (err) {
		console.error("[video/probe] Error:", err);

		if (isBusyError(err)) {
			return c.json(
				{
					error: "Server is busy",
					code: "SERVER_BUSY",
					details: "Too many concurrent requests, please retry later",
				},
				503,
			);
		}

		if (isTimeoutError(err)) {
			return c.json(
				{
					error: "Request timed out",
					code: "TIMEOUT",
					details: err instanceof Error ? err.message : String(err),
				},
				504,
			);
		}

		return c.json(
			{
				error: "Failed to probe video",
				code: PROBE_ERROR_CODE,
				details: err instanceof Error ? err.message : String(err),
			},
			500,
		);
	}
});

video.post("/thumbnail", async (c) => {
	if (!validateMediaServerSecret(c)) {
		return c.json({ error: "Unauthorized" }, 401);
	}

	const body = await c.req.json();
	const result = thumbnailSchema.safeParse(body);

	if (!result.success) {
		return c.json(
			{
				error: "Invalid request",
				code: "INVALID_REQUEST",
				details: result.error.message,
			},
			400,
		);
	}

	if (isDraining()) {
		c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
		return c.json(DRAINING_RESPONSE_BODY, 503);
	}

	try {
		const metadata = await probeVideo(result.data.videoUrl);

		const thumbnailData = await generateThumbnail(
			result.data.videoUrl,
			metadata.duration,
			{
				timestamp: result.data.timestamp,
				width: result.data.width,
				height: result.data.height,
				quality: result.data.quality,
			},
		);

		return new Response(Buffer.from(thumbnailData), {
			headers: {
				"Content-Type": "image/jpeg",
				"Content-Length": thumbnailData.length.toString(),
			},
		});
	} catch (err) {
		console.error("[video/thumbnail] Error:", err);

		if (isBusyError(err)) {
			return c.json(
				{
					error: "Server is busy",
					code: "SERVER_BUSY",
					details: "Too many concurrent requests, please retry later",
				},
				503,
			);
		}

		if (isTimeoutError(err)) {
			return c.json(
				{
					error: "Request timed out",
					code: "TIMEOUT",
					details: err instanceof Error ? err.message : String(err),
				},
				504,
			);
		}

		return c.json(
			{
				error: "Failed to generate thumbnail",
				code: MEDIA_ENGINE_ERROR_CODE,
				details: err instanceof Error ? err.message : String(err),
			},
			500,
		);
	}
});

video.post("/convert", async (c) => {
	if (!validateMediaServerSecret(c)) {
		return c.json({ error: "Unauthorized" }, 401);
	}

	const body = await c.req.json();
	const result = convertSchema.safeParse(body);

	if (!result.success) {
		return c.json(
			{
				error: "Invalid request",
				code: "INVALID_REQUEST",
				details: result.error.message,
			},
			400,
		);
	}

	let inputTempFile: TempFileHandle | null = null;
	let outputTempFile: TempFileHandle | null = null;
	let slot: VideoProcessSlot | null = null;
	const requestId = crypto.randomUUID();
	const startedAt = Date.now();

	if (isDraining()) {
		c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
		return c.json(
			{
				...getBusyResponseBody(getVideoCapacitySnapshot()),
				details: DRAINING_DETAILS,
			},
			503,
		);
	}

	try {
		slot = tryAcquireDirectVideoProcessSlot(canAcceptNewVideoProcess);
		const capacity = getVideoCapacitySnapshot();
		if (!slot) {
			c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
			logVideoEvent("video_convert_rejected", {
				requestId,
				reason: "capacity",
				...capacity,
			});
			return c.json(getBusyResponseBody(capacity), 503);
		}

		logVideoEvent("video_convert_started", {
			requestId,
			input: summarizeVideoInput(
				result.data.videoUrl,
				result.data.inputExtension,
			),
			...capacity,
		});
		setRequestIdleTimeout(c.env, c.req.raw, 0);

		inputTempFile = await downloadVideoToTemp(
			result.data.videoUrl,
			result.data.inputExtension,
			c.req.raw.signal,
		);

		const metadata = await probeVideoFile(inputTempFile.path);
		outputTempFile = await processVideo(
			inputTempFile.path,
			metadata,
			{
				maxWidth: metadata.width > 0 ? metadata.width : undefined,
				maxHeight: metadata.height > 0 ? metadata.height : undefined,
			},
			undefined,
			c.req.raw.signal,
		);

		const outputSize = await file(outputTempFile.path).size;
		logVideoEvent("video_convert_succeeded", {
			requestId,
			durationMs: Date.now() - startedAt,
			outputSize,
			metadata: {
				duration: metadata.duration,
				width: metadata.width,
				height: metadata.height,
				videoCodec: metadata.videoCodec,
				audioCodec: metadata.audioCodec,
				fileSize: metadata.fileSize,
			},
			...getVideoCapacitySnapshot(),
		});

		setRequestIdleTimeout(c.env, c.req.raw, getIdleTimeoutSeconds());
		return await createVideoDownloadResponse(
			outputTempFile,
			[inputTempFile, outputTempFile],
			slot.release,
		);
	} catch (err) {
		setRequestIdleTimeout(c.env, c.req.raw, getIdleTimeoutSeconds());
		slot?.release();
		await cleanupTempFiles([outputTempFile, inputTempFile]);
		logVideoEvent("video_convert_failed", {
			requestId,
			durationMs: Date.now() - startedAt,
			error: err instanceof Error ? err.message : String(err),
			...getVideoCapacitySnapshot(),
		});
		console.error("[video/convert] Error:", err);

		if (isBusyError(err)) {
			c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
			return c.json(
				{
					error: "Server is busy",
					code: "SERVER_BUSY",
					details: "Too many concurrent requests, please retry later",
				},
				503,
			);
		}

		if (isTimeoutError(err)) {
			return c.json(
				{
					error: "Request timed out",
					code: "TIMEOUT",
					details: err instanceof Error ? err.message : String(err),
				},
				504,
			);
		}

		return c.json(
			{
				error: "Failed to convert video",
				code: MEDIA_ENGINE_ERROR_CODE,
				details: err instanceof Error ? err.message : String(err),
			},
			500,
		);
	}
});

video.on("POST", ["/process", "/import"], async (c) => {
	if (!validateMediaServerSecret(c)) {
		return c.json({ error: "Unauthorized" }, 401);
	}

	const body = await c.req.json();
	const result = (
		c.req.path.endsWith("/import")
			? processSchema.refine(
					(input) => input.sourcePresignedUrl || input.sourceUpload,
					{ message: "An original video upload target is required" },
				)
			: processSchema
	).safeParse(body);

	if (!result.success) {
		return c.json(
			{
				error: "Invalid request",
				code: "INVALID_REQUEST",
				details: result.error.message,
			},
			400,
		);
	}

	if (
		result.data.existingOutput &&
		(result.data.sourceUpload ||
			result.data.sourcePresignedUrl ||
			result.data.outputUpload ||
			result.data.maxImportBytes !== LOOM_IMPORT_MAX_BYTES)
	) {
		return c.json(
			{
				error: "Existing output cannot be combined with upload targets",
				code: "INVALID_REQUEST",
			},
			400,
		);
	}
	const capacity = getVideoCapacitySnapshot();
	if (isDraining()) {
		c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
		return c.json(
			{ ...getBusyResponseBody(capacity), details: DRAINING_DETAILS },
			503,
		);
	}
	const bulkCapacityReached =
		result.data.priority === "bulk" &&
		capacity.effectiveMaxVideoProcesses > 1 &&
		capacity.activeVideoProcesses >= capacity.effectiveMaxVideoProcesses - 1;
	if (bulkCapacityReached || !canAcceptNewVideoProcess()) {
		c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
		return c.json(getBusyResponseBody(capacity), 503);
	}

	const {
		videoId,
		userId,
		videoUrl,
		outputPresignedUrl,
		thumbnailPresignedUrl,
		previewGifPresignedUrl,
		webhookUrl,
		webhookSecret,
	} = result.data;

	const jobId = generateJobId();
	const job = createJob(jobId, videoId, userId, webhookUrl, webhookSecret);

	void trackBackgroundTask(
		withMediaTransfers(32 * 1024 ** 3, () =>
			processVideoAsync(
				job.jobId,
				videoUrl,
				outputPresignedUrl,
				thumbnailPresignedUrl,
				previewGifPresignedUrl,
				result.data,
			),
		).catch((err) => {
			console.error(
				`[video/process] Async processing error for job ${jobId}:`,
				err,
			);
			const currentJob = getJob(jobId);
			if (
				currentJob &&
				currentJob.phase !== "error" &&
				currentJob.phase !== "complete" &&
				currentJob.phase !== "cancelled"
			) {
				updateJob(jobId, {
					phase: "error",
					error: err instanceof Error ? err.message : String(err),
					message: "Processing failed (unhandled)",
				});
			}
		}),
	);

	return c.json({
		jobId,
		status: "queued",
		message: "Video processing started",
	});
});

video.post("/edit", async (c) => {
	if (!validateMediaServerSecret(c)) {
		return c.json({ error: "Unauthorized" }, 401);
	}

	let body: unknown;
	try {
		body = await c.req.json();
	} catch {
		return c.json({ error: "Invalid request", code: "INVALID_REQUEST" }, 400);
	}
	const result = editSchema.safeParse(body);

	if (!result.success) {
		return c.json(
			{
				error: "Invalid request",
				code: "INVALID_REQUEST",
				details: result.error.message,
			},
			400,
		);
	}

	if (isDraining()) {
		c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
		return c.json(
			{
				...getBusyResponseBody(getVideoCapacitySnapshot()),
				details: DRAINING_DETAILS,
			},
			503,
		);
	}

	if (!canAcceptNewVideoProcess()) {
		c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
		return c.json(getBusyResponseBody(getVideoCapacitySnapshot()), 503);
	}

	const {
		videoId,
		userId,
		sourceUrl,
		outputPresignedUrl,
		thumbnailPresignedUrl,
		previewGifPresignedUrl,
		webhookUrl,
		webhookSecret,
	} = result.data;

	const jobId = generateJobId();
	const job = createJob(jobId, videoId, userId, webhookUrl, webhookSecret);

	void trackBackgroundTask(
		withMediaTransfers(32 * 1024 ** 3, () =>
			editVideoAsync(
				job.jobId,
				sourceUrl,
				outputPresignedUrl,
				thumbnailPresignedUrl,
				previewGifPresignedUrl,
				result.data,
			),
		).catch((err) => {
			console.error(`[video/edit] Async edit error for job ${jobId}:`, err);
			const currentJob = getJob(jobId);
			if (
				currentJob &&
				currentJob.phase !== "error" &&
				currentJob.phase !== "complete" &&
				currentJob.phase !== "cancelled"
			) {
				updateJob(jobId, {
					phase: "error",
					error: err instanceof Error ? err.message : String(err),
					message: "Edit failed (unhandled)",
				});
			}
		}),
	);

	return c.json({
		jobId,
		status: "queued",
		message: "Video edit started",
	});
});

function isWebmInput(extension: string | undefined): boolean {
	if (!extension) return false;
	const normalized = extension.toLowerCase().replace(/^\./, "");
	return normalized === "webm";
}

function needsContainerRepair(metadata: VideoMetadata): boolean {
	return (
		metadata.duration <= 0 || metadata.width === 0 || metadata.height === 0
	);
}

const RESILIENT_FLAGS: ResilientInputFlags = {
	errDetectIgnoreErr: true,
	genPts: true,
	discardCorrupt: true,
	maxMuxingQueueSize: 1024,
};

async function probeWithRepairFallback(
	inputPath: string,
	isWebm: boolean,
	abortSignal: AbortSignal,
): Promise<{ metadata: VideoMetadata; repairedFile: TempFileHandle | null }> {
	let probeError: unknown = null;
	let metadata: VideoMetadata | null = null;

	try {
		metadata = await probeVideoFile(inputPath);
	} catch (err) {
		probeError = err;
		console.warn(
			`[probeWithRepairFallback] Initial probe failed: ${err instanceof Error ? err.message : String(err)}`,
		);
	}

	if (metadata && !needsContainerRepair(metadata)) {
		return { metadata, repairedFile: null };
	}

	if (!isWebm) {
		if (probeError) throw probeError;
		if (metadata) return { metadata, repairedFile: null };
		throw new Error("Probe returned no metadata");
	}

	console.log(
		`[probeWithRepairFallback] Attempting container repair (probe ${probeError ? "failed" : `returned duration=${metadata?.duration}`})`,
	);

	const repairedFile = await repairContainer(inputPath, abortSignal);

	try {
		const repairedMetadata = await probeVideoFile(repairedFile.path);

		if (repairedMetadata.duration <= 0 && metadata && metadata.duration > 0) {
			console.log(
				"[probeWithRepairFallback] Repaired file has worse duration; using original metadata with repaired file",
			);
			return { metadata, repairedFile };
		}

		console.log(
			`[probeWithRepairFallback] Repair successful: duration=${repairedMetadata.duration}, ${repairedMetadata.width}x${repairedMetadata.height}`,
		);
		return { metadata: repairedMetadata, repairedFile };
	} catch (reProbeErr) {
		console.error(
			`[probeWithRepairFallback] Re-probe after repair also failed: ${reProbeErr instanceof Error ? reProbeErr.message : String(reProbeErr)}`,
		);
		await repairedFile.cleanup();

		if (metadata) {
			return { metadata, repairedFile: null };
		}

		throw probeError ?? reProbeErr;
	}
}

async function processWithResilientRetry(
	inputPath: string,
	originalInputPath: string,
	metadata: VideoMetadata,
	options: z.infer<typeof processSchema>,
	isWebm: boolean,
	jobId: string,
	abortSignal: AbortSignal,
): Promise<{
	outputFile: TempFileHandle;
	lastResortRepairFile: TempFileHandle | null;
}> {
	const processOptions = {
		maxWidth: options.maxWidth,
		maxHeight: options.maxHeight,
		crf: options.crf,
		preset: options.preset,
		remuxOnly: options.remuxOnly,
		normalizeH264Level: options.normalizeH264Level,
	};

	const onProgress = (progress: number, message: string) => {
		const scaledProgress = 10 + progress * 0.7;
		updateJob(jobId, { progress: scaledProgress, message });
		const currentJob = getJob(jobId);
		if (currentJob) {
			void sendWebhook(currentJob).catch((error) =>
				console.warn(
					`[video/process] Failed to send webhook update for job ${jobId}:`,
					error,
				),
			);
		}
	};

	return await withJobHeartbeat(jobId, async () => {
		try {
			const outputFile = await processVideo(
				inputPath,
				metadata,
				processOptions,
				onProgress,
				abortSignal,
			);
			return { outputFile, lastResortRepairFile: null };
		} catch (firstError) {
			if (!isWebm) throw firstError;

			console.warn(
				`[processWithResilientRetry] First transcode attempt failed: ${firstError instanceof Error ? firstError.message : String(firstError)}`,
			);

			updateJob(jobId, {
				progress: 10,
				message: "Retrying with error recovery...",
			});
			extendJobLifetimeForWork(
				jobId,
				getProcessTimeoutMs(metadata.duration, PROCESS_TIMEOUT_MS),
				metadata.fileSize,
			);

			try {
				const outputFile = await processVideo(
					inputPath,
					metadata,
					processOptions,
					onProgress,
					abortSignal,
					RESILIENT_FLAGS,
				);
				return { outputFile, lastResortRepairFile: null };
			} catch (retryError) {
				console.warn(
					`[processWithResilientRetry] Resilient retry also failed: ${retryError instanceof Error ? retryError.message : String(retryError)}`,
				);
			}

			console.log(
				"[processWithResilientRetry] Attempting last-resort container repair and transcode...",
			);

			updateJob(jobId, {
				progress: 10,
				message: "Attempting full repair...",
			});
			extendJobLifetimeForWork(
				jobId,
				getProcessTimeoutMs(metadata.duration, PROCESS_TIMEOUT_MS),
				metadata.fileSize,
			);

			let lastResortRepairFile: TempFileHandle | null = null;
			try {
				lastResortRepairFile = await repairContainer(
					originalInputPath,
					abortSignal,
				);

				let repairedMetadata: VideoMetadata;
				try {
					repairedMetadata = await probeVideoFile(lastResortRepairFile.path);
				} catch {
					repairedMetadata = metadata;
				}

				const outputFile = await processVideo(
					lastResortRepairFile.path,
					repairedMetadata,
					processOptions,
					onProgress,
					abortSignal,
					RESILIENT_FLAGS,
				);
				return { outputFile, lastResortRepairFile };
			} catch (lastResortError) {
				console.error(
					`[processWithResilientRetry] Last-resort repair+transcode failed: ${lastResortError instanceof Error ? lastResortError.message : String(lastResortError)}`,
				);
				await lastResortRepairFile?.cleanup();
				throw lastResortError;
			}
		}
	});
}

async function generateAndUploadPreviewGif(
	inputPath: string,
	duration: number,
	previewGifPresignedUrl: string | undefined,
	abortSignal: AbortSignal | undefined,
	logPrefix: string,
): Promise<void> {
	if (!previewGifPresignedUrl) return;

	const render = generatePreviewGif(inputPath, duration, {}, abortSignal);
	try {
		await uploadRenderedPreviewGif(
			render,
			previewGifPresignedUrl,
			abortSignal,
			logPrefix,
		);
	} finally {
		await render.then(
			(gif) => gif.cleanup(),
			() => {},
		);
	}
}

async function uploadRenderedPreviewGif(
	render: Promise<TempFileHandle>,
	previewGifPresignedUrl: string,
	abortSignal: AbortSignal | undefined,
	logPrefix: string,
): Promise<void> {
	try {
		const previewGifFile = await render;
		await uploadFileToS3(
			previewGifFile.path,
			previewGifPresignedUrl,
			"image/gif",
			abortSignal,
		);
	} catch (previewErr) {
		if (abortSignal?.aborted) {
			throw previewErr instanceof Error
				? previewErr
				: new Error("Preview GIF generation aborted");
		}
		console.warn(`[${logPrefix}] Preview GIF generation failed:`, previewErr);
	}
}

const UPLOAD_VERIFICATION_ATTEMPTS = 4;
const UPLOAD_VERIFICATION_RETRY_MS = 1000;

function getDurationTolerance(duration: number) {
	if (!Number.isFinite(duration) || duration <= 0) return 0.5;
	return Math.max(0.5, Math.min(5, duration * 0.01));
}

function isDurationClose(actual: number, expected: number) {
	return (
		Number.isFinite(actual) &&
		Number.isFinite(expected) &&
		Math.abs(actual - expected) <= getDurationTolerance(expected)
	);
}

async function waitForVerificationRetry(
	attempt: number,
	abortSignal?: AbortSignal,
) {
	await Bun.sleep(UPLOAD_VERIFICATION_RETRY_MS * (attempt + 1));
	abortSignal?.throwIfAborted();
}

async function verifyUploadedVideo(
	outputVerificationUrl: string | undefined,
	expectedMetadata: VideoMetadata,
	abortSignal?: AbortSignal,
) {
	if (!outputVerificationUrl) return;

	let lastError: Error | undefined;

	for (let attempt = 0; attempt < UPLOAD_VERIFICATION_ATTEMPTS; attempt++) {
		abortSignal?.throwIfAborted();
		try {
			const actualMetadata = await probeVideo(outputVerificationUrl);
			if (isDurationClose(actualMetadata.duration, expectedMetadata.duration)) {
				return;
			}

			lastError = new Error(
				`Uploaded video duration mismatch: expected ${expectedMetadata.duration.toFixed(3)}s, got ${actualMetadata.duration.toFixed(3)}s`,
			);
		} catch (error) {
			lastError = error instanceof Error ? error : new Error(String(error));
		}

		if (attempt < UPLOAD_VERIFICATION_ATTEMPTS - 1) {
			await waitForVerificationRetry(attempt, abortSignal);
		}
	}

	throw lastError ?? new Error("Uploaded video verification failed");
}

async function editVideoAsync(
	jobId: string,
	sourceUrl: string,
	outputPresignedUrl: string,
	thumbnailPresignedUrl: string | undefined,
	previewGifPresignedUrl: string | undefined,
	options: z.infer<typeof editSchema>,
): Promise<void> {
	if (!getJob(jobId)) {
		return;
	}

	const abortController = new AbortController();
	const signal = abortController.signal;
	updateJob(jobId, { abortController });
	let inputTempFile: TempFileHandle | null = null;
	let outputTempFile: TempFileHandle | null = null;

	try {
		updateJob(jobId, {
			phase: "downloading",
			progress: 0,
			message: "Downloading source video...",
		});
		const downloadingJob = getJob(jobId);
		if (downloadingJob) {
			await sendWebhook(downloadingJob);
		}

		const input = await withJobHeartbeat(jobId, () =>
			downloadVideoToTemp(sourceUrl, ".mp4", signal),
		);
		inputTempFile = input;
		updateJob(jobId, { inputTempFile: input });

		updateJob(jobId, {
			phase: "probing",
			progress: 5,
			message: "Analyzing source video...",
		});
		const probingJob = getJob(jobId);
		if (probingJob) {
			await sendWebhook(probingJob);
		}

		const sourceMetadata = await withJobHeartbeat(jobId, () =>
			probeVideoFile(input.path),
		);
		signal.throwIfAborted();
		extendJobLifetimeForWork(
			jobId,
			getEditRenderTimeoutMs(options.keepRanges),
			sourceMetadata.fileSize,
		);

		updateJob(jobId, {
			phase: "processing",
			progress: 10,
			message: "Applying edit...",
		});
		const processingJob = getJob(jobId);
		if (processingJob) {
			await sendWebhook(processingJob);
		}

		const output = await withJobHeartbeat(jobId, () =>
			renderEditedVideo({
				inputPath: input.path,
				keepRanges: options.keepRanges,
				metadata: sourceMetadata,
				abortSignal: signal,
				onProgress: (progress, message) => {
					updateJob(jobId, {
						progress: Math.min(80, 10 + progress * 0.9),
						message,
					});
					const currentJob = getJob(jobId);
					if (currentJob) {
						void sendWebhook(currentJob).catch((error) =>
							console.warn(
								`[video/edit] Failed to send webhook update for job ${jobId}:`,
								error,
							),
						);
					}
				},
			}),
		);
		outputTempFile = output;
		updateJob(jobId, { outputTempFile: output });

		const outputMetadata = await withJobHeartbeat(jobId, () =>
			probeVideoFile(output.path),
		);
		signal.throwIfAborted();
		updateJob(jobId, { metadata: outputMetadata });

		updateJob(jobId, {
			phase: "uploading",
			progress: 80,
			message: "Uploading edited video...",
		});
		const uploadingJob = getJob(jobId);
		if (uploadingJob) {
			await sendWebhook(uploadingJob);
		}

		await withJobHeartbeat(jobId, async () => {
			await uploadFileToS3(
				output.path,
				outputPresignedUrl,
				"video/mp4",
				signal,
			);
			await verifyUploadedVideo(
				options.outputVerificationUrl,
				outputMetadata,
				signal,
			);
		});

		if (thumbnailPresignedUrl || previewGifPresignedUrl) {
			updateJob(jobId, {
				phase: "generating_thumbnail",
				progress: 90,
				message: "Generating preview assets...",
			});
			const thumbnailJob = getJob(jobId);
			if (thumbnailJob) {
				await sendWebhook(thumbnailJob);
			}
		}

		await withJobHeartbeat(jobId, async () => {
			if (thumbnailPresignedUrl) {
				const thumbnailData = await generateThumbnail(
					output.path,
					outputMetadata.duration,
					{},
					signal,
				);
				await uploadToS3(
					thumbnailData,
					thumbnailPresignedUrl,
					"image/jpeg",
					signal,
				);
			}

			await generateAndUploadPreviewGif(
				output.path,
				outputMetadata.duration,
				previewGifPresignedUrl,
				signal,
				"video/edit",
			);
		});
		signal.throwIfAborted();

		updateJob(jobId, {
			phase: "complete",
			progress: 100,
			message: "Edit complete",
		});
		const completedJob = getJob(jobId);
		if (completedJob) {
			await sendWebhook(completedJob);
		}

		setTimeout(() => deleteJob(jobId), 5 * 60 * 1000);
	} catch (err) {
		if (!signal.aborted) {
			console.error(`[video/edit] Error editing job ${jobId}:`, err);
		}

		const updatedJob = updateJob(jobId, {
			phase: "error",
			error: err instanceof Error ? err.message : String(err),
			message: "Edit failed",
		});

		try {
			if (updatedJob) {
				await sendWebhook(updatedJob);
			}
		} finally {
			setTimeout(() => deleteJob(jobId), 5 * 60 * 1000);
		}
	} finally {
		await cleanupTempFiles([inputTempFile, outputTempFile]);
	}
}

async function processVideoAsync(
	jobId: string,
	videoUrl: string,
	outputPresignedUrl: string,
	thumbnailPresignedUrl: string | undefined,
	previewGifPresignedUrl: string | undefined,
	options: z.infer<typeof processSchema>,
): Promise<void> {
	const job = getJob(jobId);
	if (!job) {
		return;
	}

	const abortController = new AbortController();
	const signal = abortController.signal;
	updateJob(jobId, { abortController });

	let inputTempFile: TempFileHandle | null = null;
	let outputTempFile: TempFileHandle | null = null;
	let repairedTempFile: TempFileHandle | null = null;
	let lastResortRepairFile: TempFileHandle | null = null;
	let sourceUploaded = false;
	let outputUploaded = false;
	const abortPendingUploads = async () => {
		const targets = [
			!sourceUploaded ? options.sourceUpload : undefined,
			!outputUploaded ? options.outputUpload : undefined,
		].filter((target): target is StorageUploadTarget => target !== undefined);
		await Promise.all(
			targets.map((target) =>
				abortStorageUploadTarget(target).catch((error) => {
					console.warn("Loom multipart cleanup failed", error);
				}),
			),
		);
	};

	try {
		updateJob(jobId, {
			phase: "downloading",
			progress: 0,
			message: "Downloading video...",
		});
		// Progress webhooks only update the upload row, so don't wait on cap.so between stages. sendWebhook keeps one
		// delivery in flight per job and always sends the latest state, so the awaited terminal webhook still lands last.
		void sendWebhook(job);

		const input = await withJobHeartbeat(jobId, () =>
			downloadVideoToTemp(
				options.existingOutput?.url ?? videoUrl,
				options.inputExtension,
				signal,
				options.maxImportBytes,
			),
		);
		inputTempFile = input;
		updateJob(jobId, { inputTempFile: input });
		if (options.existingOutput) {
			const existing = options.existingOutput;
			await withJobHeartbeat(jobId, async () => {
				await verifyRecording(input.path, {
					requireAudio: false,
					allowObservedDuration: true,
					abortSignal: signal,
				});
				await verifyRemoteRecordingBytes(existing.url, {
					expectedObjectIdentity: existing.objectIdentity,
					expectedFileSize: existing.fileSize,
					expectedSha256: await hashRecordingFile(input.path, signal),
					abortSignal: signal,
				});
			});
		}

		const sourcePresignedUrl = options.sourcePresignedUrl;
		if (sourcePresignedUrl || options.sourceUpload) {
			updateJob(jobId, { message: "Saving original video..." });
			void sendWebhook(job);
			await withJobHeartbeat(jobId, () =>
				options.sourceUpload
					? uploadFileToStorage(
							input.path,
							options.sourceUpload,
							"video/mp4",
							signal,
						)
					: uploadFileToS3(
							input.path,
							sourcePresignedUrl as string,
							"video/mp4",
							signal,
						),
			);
			sourceUploaded = true;
		}

		const isWebm = isWebmInput(options.inputExtension);
		if (isWebm && !options.importedOriginal) {
			updateJob(jobId, { message: "Checking the original recording..." });
			await withJobHeartbeat(jobId, () =>
				validateVideoInput(input.path, signal),
			);
		}

		updateJob(jobId, {
			phase: "probing",
			progress: 5,
			message: "Analyzing video...",
		});
		void sendWebhook(job);

		const { metadata, repairedFile } = await withJobHeartbeat(jobId, () =>
			options.existingOutput
				? probeVideoFile(input.path).then((metadata) => ({
						metadata,
						repairedFile: null,
					}))
				: probeWithRepairFallback(input.path, isWebm, signal),
		);
		repairedTempFile = repairedFile;
		signal.throwIfAborted();
		updateJob(jobId, { metadata });
		extendJobLifetimeForWork(
			jobId,
			getProcessTimeoutMs(metadata.duration, PROCESS_TIMEOUT_MS),
			metadata.fileSize,
		);

		const processingInputPath = repairedFile ? repairedFile.path : input.path;

		updateJob(jobId, {
			phase: "processing",
			progress: 10,
			message: repairedFile
				? "Processing repaired video..."
				: "Processing video...",
		});
		void sendWebhook(job);

		const processed = options.existingOutput
			? { outputFile: input, lastResortRepairFile: null }
			: await processWithResilientRetry(
					processingInputPath,
					input.path,
					metadata,
					options,
					isWebm,
					jobId,
					signal,
				);
		outputTempFile = processed.outputFile;
		lastResortRepairFile = processed.lastResortRepairFile;
		const outputPath = processed.outputFile.path;
		updateJob(jobId, { outputTempFile: processed.outputFile });
		if (options.importedOriginal) {
			const recovered = await withJobHeartbeat(jobId, () =>
				probeVideoFile(outputPath),
			);
			if (!isCompleteRecovery(recovered.duration, metadata.duration))
				throw new Error(
					"The imported original could not be fully recovered. The original has been preserved.",
				);
		}
		if (
			options.maxImportBytes &&
			file(outputPath).size > options.maxImportBytes
		) {
			throw new Error("Loom imports are limited to 10 GB");
		}

		updateJob(jobId, {
			phase: "uploading",
			progress: 80,
			message: "Uploading processed video...",
		});
		void sendWebhook(job);

		// Previews only read the finished output, so render them while it uploads; they are still
		// uploaded after the output, so a failed upload never leaves previews behind.
		let uploadReceipt: StorageUploadReceipt;
		const previewAbort = new AbortController();
		const previewSignal = AbortSignal.any([signal, previewAbort.signal]);
		const thumbnailRender = thumbnailPresignedUrl
			? generateThumbnail(outputPath, metadata.duration, {}, previewSignal)
			: undefined;
		const previewGifRender = previewGifPresignedUrl
			? generatePreviewGif(outputPath, metadata.duration, {}, previewSignal)
			: undefined;
		thumbnailRender?.catch(() => {});
		previewGifRender?.catch(() => {});
		try {
			uploadReceipt = await withJobHeartbeat(jobId, async () =>
				options.existingOutput
					? { objectIdentity: options.existingOutput.objectIdentity }
					: options.outputUpload
						? uploadFileToStorage(
								outputPath,
								options.outputUpload,
								"video/mp4",
								signal,
							)
						: uploadFileToS3(
								outputPath,
								outputPresignedUrl,
								"video/mp4",
								signal,
							),
			);
			outputUploaded = true;

			if (thumbnailPresignedUrl || previewGifPresignedUrl) {
				updateJob(jobId, {
					phase: "generating_thumbnail",
					progress: 90,
					message: "Generating preview assets...",
				});
				void sendWebhook(job);
			}

			await withJobHeartbeat(jobId, async () => {
				if (thumbnailRender && thumbnailPresignedUrl) {
					// Only the fully verified reuse path can finish without a decoded preview.
					const thumbnailData = await thumbnailRender.catch((error) => {
						signal.throwIfAborted();
						if (!options.existingOutput) throw error;
						console.warn(
							"Verified Loom output preview could not be generated",
							error,
						);
						return undefined;
					});
					try {
						if (thumbnailData)
							await uploadToS3(
								thumbnailData,
								thumbnailPresignedUrl,
								"image/jpeg",
								signal,
							);
					} catch (error) {
						signal.throwIfAborted();
						console.warn(
							`[video/process] Thumbnail upload failed for ${jobId}:`,
							error,
						);
					}
				}

				if (previewGifRender && previewGifPresignedUrl)
					await uploadRenderedPreviewGif(
						previewGifRender,
						previewGifPresignedUrl,
						signal,
						"video/process",
					);
			});
		} finally {
			previewAbort.abort();
			await Promise.allSettled([
				thumbnailRender,
				previewGifRender?.then((gif) => gif.cleanup()),
			]);
		}
		signal.throwIfAborted();

		updateJob(jobId, {
			phase: "complete",
			progress: 100,
			message: "Processing complete",
		});
		const completedJob = getJob(jobId);
		if (completedJob) {
			await sendWebhook(completedJob);
		}

		if (options.audioLevels)
			await enhanceLocalRecording({
				path: outputPath,
				videoId: options.videoId,
				userId: options.userId,
				jobId,
				sourceKey: `${options.userId}/${options.videoId}/result.mp4`,
				sourceIdentity: uploadReceipt?.objectIdentity,
				duration: metadata.duration,
				webhookUrl: options.webhookUrl,
				webhookSecret: options.webhookSecret,
			});

		setTimeout(() => deleteJob(jobId), 5 * 60 * 1000);
	} catch (err) {
		await abortPendingUploads();
		if (!signal.aborted) {
			console.error(`[video/process] Error processing job ${jobId}:`, err);
		}
		const failedJob = getJob(jobId);
		if (
			failedJob &&
			failedJob.phase !== "complete" &&
			failedJob.phase !== "cancelled" &&
			failedJob.phase !== "error"
		) {
			const cancelled = signal.aborted;
			const updatedJob = updateJob(jobId, {
				phase: cancelled ? "cancelled" : "error",
				error: cancelled
					? undefined
					: err instanceof Error
						? err.message
						: String(err),
				message: cancelled ? "Processing cancelled" : "Processing failed",
			});
			if (updatedJob) {
				await sendWebhook(updatedJob);
			}
		}
	} finally {
		await cleanupTempFiles([
			inputTempFile,
			outputTempFile,
			repairedTempFile,
			lastResortRepairFile,
		]);
	}
}

video.get("/process/:jobId/status", async (c) => {
	if (!canReadStatus(c)) {
		return c.json({ error: "Unauthorized" }, 401);
	}
	const jobId = c.req.param("jobId");
	const job = getJob(jobId);
	const accept = c.req.header("Accept");

	if (!job) {
		if (
			hasMediaServerSecret(c) &&
			!isPeerLookup(c.req.raw) &&
			!accept?.includes("text/event-stream")
		) {
			const owner = await findJobOnPeer(
				`/video/process/${encodeURIComponent(jobId)}/status`,
				"GET",
			);
			if (owner) return owner;
		}
		return c.json(
			{
				error: "Job not found",
				code: "NOT_FOUND",
				instanceId: getInstanceId(),
				pid: process.pid,
			},
			404,
		);
	}

	if (accept?.includes("text/event-stream")) {
		const stream = new ReadableStream({
			start(controller) {
				const encoder = new TextEncoder();

				const sendUpdate = () => {
					const currentJob = getJob(jobId);
					if (!currentJob) {
						controller.enqueue(
							encoder.encode(
								`data: ${JSON.stringify({ error: "Job not found" })}\n\n`,
							),
						);
						controller.close();
						return false;
					}

					const progress = getJobProgress(currentJob);
					controller.enqueue(
						encoder.encode(`data: ${JSON.stringify(progress)}\n\n`),
					);

					if (
						currentJob.phase === "complete" ||
						currentJob.phase === "error" ||
						currentJob.phase === "cancelled"
					) {
						controller.close();
						return false;
					}

					return true;
				};

				sendUpdate();

				const interval = setInterval(() => {
					if (!sendUpdate()) {
						clearInterval(interval);
					}
				}, 1000);

				c.req.raw.signal.addEventListener("abort", () => {
					clearInterval(interval);
					controller.close();
				});
			},
		});

		return new Response(stream, {
			headers: {
				"Content-Type": "text/event-stream",
				"Cache-Control": "no-cache",
				Connection: "keep-alive",
			},
		});
	}

	return c.json(getJobProgress(job));
});

video.post("/process/:jobId/cancel", async (c) => {
	if (!validateMediaServerSecret(c)) {
		return c.json({ error: "Unauthorized" }, 401);
	}
	const jobId = c.req.param("jobId");
	const job = getJob(jobId);

	if (!job) {
		if (!isPeerLookup(c.req.raw)) {
			const owner = await findJobOnPeer(
				`/video/process/${encodeURIComponent(jobId)}/cancel`,
				"POST",
			);
			if (owner) return owner;
		}
		return c.json(
			{
				error: "Job not found",
				code: "NOT_FOUND",
				instanceId: getInstanceId(),
				pid: process.pid,
			},
			404,
		);
	}

	if (
		job.phase === "complete" ||
		job.phase === "error" ||
		job.phase === "cancelled"
	) {
		return c.json(
			{
				error: "Job already finished",
				code: "INVALID_STATE",
				currentPhase: job.phase,
			},
			400,
		);
	}

	job.abortController?.abort();

	updateJob(jobId, {
		phase: "cancelled",
		message: "Processing cancelled by user",
	});

	const cancelledJob = getJob(jobId);
	if (cancelledJob) {
		await sendWebhook(cancelledJob);
	}

	return c.json({
		success: true,
		message: "Job cancelled",
	});
});

video.post("/cleanup", async (c) => {
	if (!validateMediaServerSecret(c)) {
		return c.json({ error: "Unauthorized" }, 401);
	}
	const cleaned = await cleanupStaleTempFiles();
	return c.json({
		success: true,
		cleanedFiles: cleaned,
	});
});

video.post("/force-cleanup", (c) => {
	if (!validateMediaServerSecret(c)) {
		return c.json({ error: "Unauthorized" }, 401);
	}
	const cleaned = forceCleanupActiveJobs();
	return c.json({
		success: true,
		cleanedJobs: cleaned,
		message: `Force-cleaned ${cleaned} active jobs`,
	});
});

const strongObjectIdentitySchema = z
	.string()
	.max(1_024)
	.regex(/^"[\x21\x23-\x7E\x80-\xFF]+"$/);
const recordingAttemptFields = {
	processingPriority: z.enum(["interactive", "recovery"]).optional(),
	downloadBudgetBytes: z.number().int().positive().safe().optional(),
	generation: z.string().min(1).max(200).optional(),
	attemptId: z.string().min(1).max(200).optional(),
	outputKey: z.string().min(1).max(1_024).optional(),
	inventorySha256: z
		.string()
		.regex(/^[a-f0-9]{64}$/)
		.optional(),
};

const recordingVerificationSchema = z
	.object({
		...recordingAttemptFields,
		videoId: z.string(),
		userId: z.string(),
		videoUrl: z.string().url(),
		fileSize: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
		duration: z.number().finite().positive().optional(),
		requiredAudio: z.boolean(),
		objectIdentity: strongObjectIdentitySchema,
		originalObjectIdentity: strongObjectIdentitySchema.optional(),
		sourceObjectIdentity: strongObjectIdentitySchema.optional(),
		webhookUrl: z.string().url(),
		webhookSecret: z.string().optional(),
	})
	.refine(
		(body) => {
			const fenced = Boolean(
				body.generation ||
					body.attemptId ||
					body.outputKey ||
					body.inventorySha256 ||
					body.originalObjectIdentity ||
					body.sourceObjectIdentity,
			);
			return fenced
				? Boolean(
						body.generation &&
							body.attemptId &&
							body.outputKey &&
							body.inventorySha256 &&
							body.originalObjectIdentity &&
							body.sourceObjectIdentity,
					)
				: body.duration !== undefined;
		},
		{ message: "Complete recording attempt context is required" },
	);

function recordingJobId(
	kind: string,
	body: {
		userId: string;
		videoId: string;
		generation?: string;
		attemptId?: string;
	},
): string {
	return body.generation && body.attemptId
		? `job_recording_${createHash("sha256")
				.update(
					JSON.stringify([
						kind,
						body.userId,
						body.videoId,
						body.generation,
						body.attemptId,
					]),
				)
				.digest("hex")}_${RECORDING_WORKER_INSTANCE}`
		: generateJobId();
}

function recordingRequestKey(values: unknown[]): string {
	return createHash("sha256").update(JSON.stringify(values)).digest("hex");
}

async function recordingWorkerOwner(job: Job): Promise<string | undefined> {
	if (!job.generation || !job.attemptId) return job.jobId;
	const acknowledgement = await claimRecordingWorker(job);
	if (acknowledgement?.status === "owned" && acknowledgement.ownerJobId) {
		return acknowledgement.ownerJobId;
	}
	if (
		acknowledgement?.status === "accepted" &&
		job.recordingWorkerClaimed &&
		!job.recordingWorkerRevoked
	)
		return job.jobId;
	return undefined;
}

function recordingWorkerResponse(job: Job, ownerJobId: string) {
	return {
		jobId: ownerJobId,
		...(job.generation && job.attemptId && { recordingWorkerVersion: 1 }),
	};
}

function classifySourceError(error: unknown): RecordingErrorCode {
	if (error instanceof MediaTransferBudgetError)
		return "processing-budget-exhausted";
	if (
		isRetryableRecordingVerificationError(error) ||
		isBusyError(error) ||
		isTimeoutError(error)
	)
		return "processing-unavailable";
	if (!(error instanceof Error)) return "processing-unavailable";
	if (
		/Cannot allocate memory|Resource temporarily unavailable|No space left|Unknown encoder|Unknown decoder|Permission denied|Function not implemented|Option not found/i.test(
			error.message,
		)
	)
		return "processing-unavailable";
	if (/object changed|HTTP error 412|Server returned 412/.test(error.message))
		return "source-changed";
	if (/HTTP error 404|Server returned 404/.test(error.message))
		return "source-missing";
	if (
		/Decoded recording|Invalid decoded recording|Recording has no decoded|Recording video packets are missing|Recording audio timing is invalid|Recording timing has no video track|does not match the completed local file|Verified recording size/.test(
			error.message,
		)
	)
		return "source-invalid";
	if (
		error.message.includes("Recording full decode failed") &&
		/invalid|corrupt|error while decoding|moov atom not found|Cannot determine format.*EOF|partial file/i.test(
			error.message,
		)
	)
		return "source-invalid";
	return "processing-unavailable";
}

video.post("/verify-recording", async (c) => {
	if (!validateMediaServerSecret(c))
		return c.json({ error: "Unauthorized" }, 401);
	const parsed = recordingVerificationSchema.safeParse(await c.req.json());
	if (!parsed.success)
		return c.json({ error: "Invalid verification request" }, 400);
	const body = parsed.data;
	const jobId = recordingJobId("mp4", body);
	const requestKey = recordingRequestKey([
		body.fileSize,
		body.duration,
		body.requiredAudio,
		body.originalObjectIdentity,
		body.sourceObjectIdentity,
		body.outputKey,
		body.inventorySha256,
	]);
	const existing = getJob(jobId);
	if (existing) {
		if (existing.recordingRequestKey !== requestKey)
			return c.json({ error: "Recording attempt context changed" }, 409);
		const ownerJobId = await recordingWorkerOwner(existing);
		if (!ownerJobId)
			return c.json(
				{
					error: "Recording worker ownership unavailable",
					code: "RECORDING_OWNERSHIP_UNAVAILABLE",
				},
				503,
			);
		return c.json(recordingWorkerResponse(existing, ownerJobId));
	}
	if (isDraining()) {
		c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
		return c.json(
			{
				...getMuxBusyResponseBody(getVideoCapacitySnapshot()),
				details: DRAINING_DETAILS,
			},
			503,
		);
	}
	if (!canAcceptNewVideoProcess(body.processingPriority)) {
		c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
		return c.json(getMuxBusyResponseBody(getVideoCapacitySnapshot()), 503);
	}
	const job = createJob(
		jobId,
		body.videoId,
		body.userId,
		body.webhookUrl,
		body.webhookSecret,
	);
	updateJob(jobId, {
		generation: body.generation,
		attemptId: body.attemptId,
		inventorySha256: body.inventorySha256,
		recordingRequestKey: requestKey,
	});
	const ownerJobId = await recordingWorkerOwner(job);
	if (!ownerJobId) {
		deleteJob(jobId);
		return c.json(
			{
				error: "Recording worker ownership unavailable",
				code: "RECORDING_OWNERSHIP_UNAVAILABLE",
			},
			503,
		);
	}
	if (ownerJobId !== jobId) {
		deleteJob(jobId);
		return c.json(recordingWorkerResponse(job, ownerJobId));
	}
	void trackBackgroundTask(
		withMediaTransfers(body.downloadBudgetBytes ?? body.fileSize * 3, () =>
			verifyUploadedRecordingAsync(jobId, body),
		).catch((error: unknown) => {
			const current = getJob(jobId);
			if (
				!current ||
				["complete", "error", "cancelled"].includes(current.phase)
			)
				return;
			const errorCode = classifySourceError(error);
			logVideoEvent("recording_verification_failed", {
				jobId,
				videoId: body.videoId,
				phase: "error",
				errorCode,
				retryable: errorCode === "processing-unavailable",
				stage: "transfer",
				reason: logReason(error),
			});
			updateJob(jobId, {
				phase: "error",
				error: "Recording transfer could not complete",
				errorCode,
			});
			sendCurrentJobWebhook(jobId);
		}),
	);
	return c.json(recordingWorkerResponse(job, jobId));
});

async function verifyUploadedRecordingAsync(
	jobId: string,
	body: z.infer<typeof recordingVerificationSchema>,
) {
	const abortController = new AbortController();
	if (!updateJob(jobId, { abortController, phase: "processing", progress: 0 }))
		return;
	let stage = "probe";
	let cause: unknown;
	let mismatch: Record<string, unknown> | undefined;
	try {
		const metadata = await probeVideo(body.videoUrl).catch((error: unknown) => {
			if (error instanceof MediaTransferBudgetError) throw error;
			cause = error;
			throw new Error(RECORDING_VERIFICATION_RETRY_ERROR);
		});
		stage = "metadata";
		if (
			metadata.fileSize !== body.fileSize ||
			(body.duration !== undefined &&
				!isDurationClose(metadata.duration, body.duration)) ||
			(body.requiredAudio && !metadata.audioCodec)
		) {
			mismatch = {
				expectedFileSize: body.fileSize,
				actualFileSize: metadata.fileSize,
				expectedDuration: body.duration,
				actualDuration: metadata.duration,
				requiredAudio: body.requiredAudio,
				audioCodec: metadata.audioCodec,
			};
			throw new Error(
				"Uploaded recording does not match the completed local file",
			);
		}
		stage = "decode";
		if (
			!beginRecordingVerification(
				jobId,
				PROCESS_TIMEOUT_MS + POST_VERIFICATION_ASSET_BUDGET_MS,
			)
		)
			throw new Error("Recording verification job is no longer active");
		const verified = await withJobHeartbeat(jobId, () =>
			withMuxMemoryGuard(abortController, () =>
				verifyRemoteRecording(body.videoUrl, {
					expectedDuration: body.duration,
					allowObservedDuration: body.duration === undefined,
					requireAudio: body.requiredAudio,
					hasAudio: metadata.audioChannels !== null,
					expectedObjectIdentity:
						body.sourceObjectIdentity ?? body.objectIdentity,
					expectedFileSize: body.fileSize,
					hashContent: Boolean(body.generation),
					hashDecodedContent: false,
					abortSignal: abortController.signal,
				}),
			),
		);
		stage = "identity";
		if (verified.fileSize !== body.fileSize)
			throw new Error("Verified recording size does not match the local file");
		if (body.generation && !verified.remoteSha256)
			throw new Error("Recording byte identity is missing");
		updateJob(jobId, {
			phase: "complete",
			progress: 100,
			metadata: {
				...metadata,
				duration: verified.video.duration,
				audioCodec: verified.audio ? (metadata.audioCodec ?? "unknown") : null,
				fileSize: verified.fileSize,
			},
			recordingVerification: {
				request: {
					version: 1,
					artifact: {
						kind: "mp4",
						fileSize: body.fileSize,
						duration: body.duration ?? verified.video.duration,
						objectIdentity: body.originalObjectIdentity ?? body.objectIdentity,
					},
					requiredAudio: body.requiredAudio,
				},
				fullDecode: true,
				objectIdentity: verified.objectIdentity,
				...(body.outputKey
					? { outputKey: body.outputKey, outputSha256: verified.remoteSha256 }
					: {}),
			},
		});
	} catch (error) {
		const retryable =
			isRetryableRecordingVerificationError(error) ||
			(error instanceof Error &&
				(error.message === VIDEO_MEMORY_PRESSURE_ERROR ||
					error.message === RECORDING_VERIFICATION_RETRY_ERROR));
		const phase =
			!retryable && abortController.signal.aborted ? "cancelled" : "error";
		const errorCode = retryable
			? "processing-unavailable"
			: classifySourceError(error);
		logVideoEvent("recording_verification_failed", {
			jobId,
			videoId: body.videoId,
			phase,
			errorCode,
			retryable: errorCode === "processing-unavailable",
			stage,
			reason: logReason(cause ?? error),
			...(mismatch && { mismatch }),
		});
		updateJob(jobId, {
			phase,
			error: retryable
				? RECORDING_VERIFICATION_RETRY_ERROR
				: "Uploaded recording content could not be verified; retain the local recording",
			errorCode,
		});
	}
	const job = getJob(jobId);
	if (job) await sendWebhook(job);
}

const muxSegmentsSchema = z
	.object({
		audioLevels: z.boolean().optional(),
		...recordingAttemptFields,
		requiredAudio: z.boolean().optional(),
		sourceObjects: z
			.array(
				z.object({
					url: z.string().url(),
					objectIdentity: strongObjectIdentitySchema,
					size: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
				}),
			)
			.max(100_002)
			.optional(),
		manifestSha256: z
			.string()
			.regex(/^[a-f0-9]{64}$/)
			.optional(),
		expectedDuration: z.number().finite().positive().optional(),
		outputVerificationUrl: z.string().url().optional(),
		videoId: z.string(),
		userId: z.string(),
		outputPresignedUrl: z.string().url().optional(),
		outputUpload: storageUploadSchema.optional(),
		thumbnailPresignedUrl: z.string().url().optional(),
		previewGifPresignedUrl: z.string().url().optional(),
		webhookUrl: z.string().url().optional(),
		webhookSecret: z.string().optional(),
		videoInitUrl: z.string().url(),
		videoSegmentUrls: z.array(z.string().url()),
		audioInitUrl: z.string().url().optional(),
		audioSegmentUrls: z.array(z.string().url()).optional(),
		preverifiedRunUrls: z.array(z.string().url()).max(4_096).optional(),
	})
	.refine((body) => body.outputPresignedUrl || body.outputUpload, {
		message: "outputPresignedUrl or outputUpload is required",
		path: ["outputUpload"],
	})
	.superRefine((body, ctx) => {
		const fenced = Boolean(
			body.generation ||
				body.attemptId ||
				body.outputKey ||
				body.inventorySha256 ||
				body.sourceObjects,
		);
		if (
			fenced &&
			!(
				body.generation &&
				body.attemptId &&
				body.outputKey &&
				body.inventorySha256 &&
				body.sourceObjects &&
				body.manifestSha256 &&
				body.outputVerificationUrl
			)
		) {
			ctx.addIssue({
				code: "custom",
				message: "Complete recording attempt context is required",
			});
		}
		if (
			Boolean(body.audioInitUrl) !== Boolean(body.audioSegmentUrls?.length) ||
			(body.requiredAudio && !body.audioSegmentUrls?.length)
		) {
			ctx.addIssue({
				code: "custom",
				message: "Required audio sources are incomplete",
			});
		}
		if (body.sourceObjects) {
			const urls = [
				body.videoInitUrl,
				...body.videoSegmentUrls,
				...(body.audioInitUrl
					? [body.audioInitUrl, ...(body.audioSegmentUrls ?? [])]
					: []),
			];
			const indexed = new Map(
				body.sourceObjects.map((source) => [source.url, source]),
			);
			if (
				indexed.size !== body.sourceObjects.length ||
				new Set(urls).size !== urls.length ||
				indexed.size !== urls.length ||
				urls.some((url) => !indexed.has(url))
			) {
				ctx.addIssue({
					code: "custom",
					message:
						"Recording source inventory does not exactly cover its inputs",
				});
			}
		}
		if (
			fenced &&
			(body.outputUpload?.type === "put"
				? body.outputUpload.ifNoneMatch !== "*"
				: body.outputUpload?.type !== "multipart" ||
					body.outputUpload.key !== body.outputKey ||
					body.outputUpload.videoId !== body.videoId ||
					body.outputUpload.generation !== body.generation ||
					body.outputUpload.attemptId !== body.attemptId)
		) {
			ctx.addIssue({
				code: "custom",
				message: "Recording candidate must use an immutable upload target",
			});
		}
	});

type MuxSourceObject = NonNullable<
	z.infer<typeof muxSegmentsSchema>["sourceObjects"]
>[number];
type MuxContext = Pick<
	z.infer<typeof muxSegmentsSchema>,
	| "generation"
	| "attemptId"
	| "outputKey"
	| "inventorySha256"
	| "sourceObjects"
	| "requiredAudio"
	| "audioLevels"
	| "preverifiedRunUrls"
>;

function getMuxSegmentsOutputUpload(
	body: z.infer<typeof muxSegmentsSchema>,
): StorageUploadTarget {
	if (body.outputUpload) return body.outputUpload;
	if (body.outputPresignedUrl) {
		return { type: "put", url: body.outputPresignedUrl };
	}
	throw new Error("Missing mux output upload target");
}

video.post("/mux-segments", async (c) => {
	if (!validateMediaServerSecret(c)) {
		return c.json({ error: "Unauthorized" }, 401);
	}

	const body = muxSegmentsSchema.safeParse(await c.req.json());
	if (!body.success) {
		return c.json(
			{ error: "Invalid request", details: body.error.issues },
			400,
		);
	}

	const {
		videoId,
		userId,
		thumbnailPresignedUrl,
		previewGifPresignedUrl,
		webhookUrl,
		webhookSecret,
	} = body.data;
	const jobId = recordingJobId("segments", body.data);
	const requestKey = recordingRequestKey([
		body.data.manifestSha256,
		body.data.inventorySha256,
		body.data.outputKey,
		body.data.requiredAudio,
	]);
	const existing = getJob(jobId);
	if (existing) {
		if (existing.recordingRequestKey !== requestKey)
			return c.json({ error: "Recording attempt context changed" }, 409);
		const ownerJobId = await recordingWorkerOwner(existing);
		if (!ownerJobId)
			return c.json(
				{
					error: "Recording worker ownership unavailable",
					code: "RECORDING_OWNERSHIP_UNAVAILABLE",
				},
				503,
			);
		return c.json({
			...recordingWorkerResponse(existing, ownerJobId),
			status: "queued",
			videoId,
		});
	}

	if (isDraining()) {
		c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
		return c.json(
			{
				...getMuxBusyResponseBody(getVideoCapacitySnapshot()),
				details: DRAINING_DETAILS,
			},
			503,
		);
	}

	if (!canAcceptNewVideoProcess(body.data.processingPriority)) {
		c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString());
		return c.json(getMuxBusyResponseBody(getVideoCapacitySnapshot()), 503);
	}

	const job = createJob(jobId, videoId, userId, webhookUrl, webhookSecret);

	const {
		videoInitUrl,
		videoSegmentUrls: videoSegUrls,
		audioInitUrl,
		audioSegmentUrls: audioSegUrls,
	} = body.data;
	const outputUpload = getMuxSegmentsOutputUpload(body.data);
	updateJob(jobId, {
		manifestSha256: body.data.manifestSha256,
		generation: body.data.generation,
		attemptId: body.data.attemptId,
		inventorySha256: body.data.inventorySha256,
		recordingRequestKey: requestKey,
	});
	const ownerJobId = await recordingWorkerOwner(job);
	if (!ownerJobId) {
		deleteJob(jobId);
		return c.json(
			{
				error: "Recording worker ownership unavailable",
				code: "RECORDING_OWNERSHIP_UNAVAILABLE",
			},
			503,
		);
	}
	if (ownerJobId !== jobId) {
		deleteJob(jobId);
		return c.json({
			...recordingWorkerResponse(job, ownerJobId),
			status: "queued",
			videoId,
		});
	}

	void trackBackgroundTask(
		withMediaTransfers(body.data.downloadBudgetBytes ?? 32 * 1024 ** 3, () =>
			muxSegmentsAsync(
				jobId,
				videoId,
				outputUpload,
				thumbnailPresignedUrl,
				previewGifPresignedUrl,
				videoInitUrl,
				videoSegUrls,
				audioInitUrl ?? null,
				audioSegUrls ?? null,
				body.data.outputVerificationUrl,
				body.data,
			),
		).catch((err) => {
			console.error(`[mux-segments] Async mux error for job ${jobId}:`, err);
			const currentJob = getJob(jobId);
			if (
				currentJob &&
				currentJob.phase !== "error" &&
				currentJob.phase !== "complete" &&
				currentJob.phase !== "cancelled"
			) {
				updateJob(jobId, {
					phase: "error",
					error: err instanceof Error ? err.message : String(err),
				});
				sendCurrentJobWebhook(jobId);
			}
		}),
	);

	return c.json({
		...recordingWorkerResponse(job, jobId),
		status: "queued",
		videoId,
	});
});

const preverifySourceSchema = z.object({
	url: z.string().url(),
	objectIdentity: strongObjectIdentitySchema,
	size: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
});
const preverifySchema = z.object({
	videoId: z.string().min(1).max(200),
	init: preverifySourceSchema,
	segments: z.array(preverifySourceSchema).min(1).max(256),
	resultUploadUrl: z.string().url(),
});
const MAX_ACTIVE_PREVERIFY_RUNS = 2;
const PREVERIFIED_RUN_LOAD_BUDGET_MS = 10_000;
const PREVERIFY_RUN_TIMEOUT_MS = 5 * 60 * 1000;
let activePreverifyRuns = 0;

// Decodes fragments while the recording is still going, so the final verification only decodes the tail.
// The work is optional: a busy or draining server declines it and the final verification does it instead.
video.post("/preverify-segments", async (c) => {
	if (!validateMediaServerSecret(c)) {
		return c.json({ error: "Unauthorized" }, 401);
	}
	const body = preverifySchema.safeParse(await c.req.json());
	if (!body.success) {
		return c.json(
			{ error: "Invalid request", details: body.error.issues },
			400,
		);
	}
	const secret = process.env.MEDIA_SERVER_WEBHOOK_SECRET;
	if (
		!secret ||
		isDraining() ||
		activePreverifyRuns >= MAX_ACTIVE_PREVERIFY_RUNS
	) {
		return c.json({ error: "Server busy", code: "SERVER_BUSY" }, 503);
	}
	activePreverifyRuns++;
	void trackBackgroundTask(
		preverifySegments(body.data, secret)
			.catch((error: unknown) => {
				logVideoEvent("recording_preverify_failed", {
					videoId: body.data.videoId,
					segments: body.data.segments.length,
					error: logReason(error, MUX_FAILURE_LOG_LENGTH),
				});
			})
			.finally(() => {
				activePreverifyRuns--;
			}),
	);
	return c.json({ status: "queued" }, 202);
});

async function preverifySegments(
	body: z.infer<typeof preverifySchema>,
	secret: string,
): Promise<void> {
	const { ensureTempDir, getTempDir, retainTempPath } = await import(
		"../lib/temp-files"
	);
	const { mkdir, rm } = await import("node:fs/promises");
	const { join } = await import("node:path");
	const startedAt = Date.now();
	const signal = AbortSignal.timeout(PREVERIFY_RUN_TIMEOUT_MS);
	await ensureTempDir();
	const workDir = join(getTempDir(), `preverify-${randomUUID()}`);
	const releaseWorkDir = retainTempPath(workDir);
	try {
		await mkdir(workDir, { recursive: true });
		const initPath = join(workDir, "init.mp4");
		const segmentPaths = body.segments.map((_, index) =>
			join(workDir, `segment-${index}.m4s`),
		);
		const downloads = [
			{ source: body.init, path: initPath },
			...body.segments.map((source, index) => ({
				source,
				path: segmentPaths[index] as string,
			})),
		];
		// One failed download stops the others, and the run only ends once none is still writing.
		const downloadAbort = new AbortController();
		const downloadSignal = AbortSignal.any([signal, downloadAbort.signal]);
		let next = 0;
		const workers = await Promise.allSettled(
			Array.from({ length: Math.min(8, downloads.length) }, async () => {
				while (next < downloads.length && !downloadSignal.aborted) {
					const download = downloads[next++];
					if (!download) return;
					try {
						await downloadUrlToFile(
							download.source.url,
							download.path,
							downloadSignal,
							download.source,
						);
					} catch (error) {
						downloadAbort.abort();
						throw error;
					}
				}
			}),
		);
		const failed = workers.find((worker) => worker.status === "rejected");
		if (failed?.status === "rejected") throw failed.reason;
		const lines = await decodePreverifiedRun(
			initPath,
			segmentPaths,
			workDir,
			signal,
		);
		if (!lines) {
			logVideoEvent("recording_preverify_skipped", {
				videoId: body.videoId,
				segments: body.segments.length,
				durationMs: Date.now() - startedAt,
			});
			return;
		}
		const run = signPreverifiedRun(
			{
				version: 1,
				init: {
					objectIdentity: body.init.objectIdentity,
					size: body.init.size,
				},
				segments: body.segments.map(({ objectIdentity, size }) => ({
					objectIdentity,
					size,
				})),
				lines,
			},
			secret,
		);
		await uploadToS3(
			new TextEncoder().encode(run),
			body.resultUploadUrl,
			"application/json",
			signal,
		);
		logVideoEvent("recording_preverify_complete", {
			videoId: body.videoId,
			segments: body.segments.length,
			frames: lines.filter((line) => !line.startsWith("#")).length,
			durationMs: Date.now() - startedAt,
		});
	} finally {
		await rm(workDir, { recursive: true, force: true }).catch(() => {});
		releaseWorkDir();
	}
}

async function loadPreverifiedRuns(
	urls: string[],
	secret: string,
	signal: AbortSignal,
): Promise<PreverifiedRunPayload[]> {
	const runs: PreverifiedRunPayload[] = [];
	// Cached runs only save decoding, so loading them never holds a job back for long.
	const budget = AbortSignal.any([
		signal,
		AbortSignal.timeout(PREVERIFIED_RUN_LOAD_BUDGET_MS),
	]);
	let next = 0;
	await Promise.all(
		Array.from({ length: Math.min(16, urls.length) }, async () => {
			while (next < urls.length && !budget.aborted) {
				const url = urls[next++];
				if (!url) return;
				try {
					const response = await fetch(url, { signal: budget });
					const text = response.ok
						? await readBoundedText(response, MAX_PREVERIFIED_RUN_BYTES)
						: null;
					if (text === null) {
						await response.body?.cancel().catch(() => {});
						continue;
					}
					const run = readPreverifiedRun(text, secret);
					if (run) runs.push(run);
				} catch {
					signal.throwIfAborted();
				}
			}
		}),
	);
	return runs;
}

async function streamConcatFiles(
	inputPaths: string[],
	outputPath: string,
	abortSignal?: AbortSignal,
): Promise<void> {
	const writer = file(outputPath).writer();
	let lastMemoryCheckAt = 0;
	try {
		for (const filePath of inputPaths) {
			const reader = file(filePath).stream().getReader();
			try {
				while (true) {
					abortSignal?.throwIfAborted();
					const { done, value } = await reader.read();
					if (done) break;
					await writer.write(value);
					await writer.flush();
					const now = Date.now();
					if (now - lastMemoryCheckAt >= 1000) {
						lastMemoryCheckAt = now;
						if (hasCriticalMemoryPressure()) {
							throw new Error(VIDEO_MEMORY_PRESSURE_ERROR);
						}
					}
				}
			} finally {
				await reader.cancel().catch(() => {});
				reader.releaseLock();
			}
		}
	} finally {
		await writer.end();
	}
}

function redactPresignedUrl(url: string): string {
	try {
		const parsed = new URL(url);
		return `${parsed.origin}${parsed.pathname}`;
	} catch {
		return url.split("?")[0] ?? url;
	}
}

class MediaDownloadError extends Error {
	constructor(
		message: string,
		readonly retryable: boolean,
		readonly errorCode: RecordingErrorCode = "processing-unavailable",
	) {
		super(message);
	}
}

function isRetryableDownloadStatus(status: number): boolean {
	return (
		status === 408 ||
		status === 425 ||
		status === 429 ||
		status === 500 ||
		status === 502 ||
		status === 503 ||
		status === 504
	);
}

async function downloadUrlToFileOnce(
	url: string,
	destPath: string,
	abortSignal?: AbortSignal,
	source?: MuxSourceObject,
): Promise<void> {
	const abortController = new AbortController();
	const timeoutSignal = AbortSignal.timeout(120_000);
	const local = await materializeMedia(
		url,
		abortSignal ? AbortSignal.any([abortSignal, timeoutSignal]) : timeoutSignal,
		source?.objectIdentity,
	);
	if (local) {
		if (source && local.target.size !== source.size)
			throw new MediaDownloadError(
				"Recording source size changed while downloading",
				false,
				"source-changed",
			);
		await link(local.path, destPath);
		await releaseMaterializedMedia(local.path);
		return;
	}
	const resp = await fetchMedia(url, {
		headers: source
			? {
					"If-Match": source.objectIdentity,
					"X-Cap-Recording-Verification": "1",
				}
			: undefined,
		signal: abortSignal
			? AbortSignal.any([abortController.signal, abortSignal, timeoutSignal])
			: AbortSignal.any([abortController.signal, timeoutSignal]),
	});
	if (!resp.ok) {
		await resp.body?.cancel().catch(() => {});
		throw new MediaDownloadError(
			`Download failed (${resp.status}): ${redactPresignedUrl(url)}`,
			isRetryableDownloadStatus(resp.status) ||
				(Boolean(source) && resp.status === 404),
			source && resp.status === 412
				? "source-changed"
				: source && resp.status === 404
					? "source-missing"
					: "processing-unavailable",
		);
	}
	if (
		source &&
		(resp.status !== 200 ||
			resp.headers.get("etag") !== source.objectIdentity ||
			(resp.headers.has("content-length") &&
				Number(resp.headers.get("content-length")) !== source.size))
	) {
		await resp.body?.cancel().catch(() => {});
		throw new MediaDownloadError(
			"Recording source object changed while downloading",
			false,
			"source-changed",
		);
	}
	if (!resp.body) {
		throw new MediaDownloadError(
			`Download returned no body: ${redactPresignedUrl(url)}`,
			true,
		);
	}

	const reader = resp.body.getReader();
	const writer = file(destPath).writer();
	let lastMemoryCheckAt = 0;
	let failure: unknown;
	let bytesRead = 0;
	try {
		while (true) {
			const { done, value } = await reader.read();
			if (done) break;
			bytesRead += value.byteLength;
			if (source && bytesRead > source.size)
				throw new MediaDownloadError(
					"Recording source size changed while downloading",
					false,
					"source-changed",
				);
			await writer.write(value);
			await writer.flush();
			const now = Date.now();
			if (now - lastMemoryCheckAt >= 1000) {
				lastMemoryCheckAt = now;
				if (hasCriticalMemoryPressure()) {
					throw new Error(VIDEO_MEMORY_PRESSURE_ERROR);
				}
			}
		}
		if (source && bytesRead !== source.size)
			throw new MediaDownloadError(
				"Recording source size changed while downloading",
				false,
				"source-changed",
			);
	} catch (error) {
		failure = error;
		abortController.abort();
		await reader.cancel().catch(() => {});
	} finally {
		reader.releaseLock();
		try {
			await writer.end();
		} catch (error) {
			failure ??= error;
		}
	}

	if (failure !== undefined) {
		const { rm } = await import("node:fs/promises");
		await rm(destPath, { force: true }).catch(() => {});
		throw failure instanceof Error
			? failure
			: new Error("Download failed while streaming response body");
	}
}

async function downloadUrlToFile(
	url: string,
	destPath: string,
	abortSignal?: AbortSignal,
	source?: MuxSourceObject,
): Promise<void> {
	let lastError: Error | undefined;

	for (let attempt = 0; attempt < SEGMENT_DOWNLOAD_MAX_ATTEMPTS; attempt++) {
		abortSignal?.throwIfAborted();
		try {
			await downloadUrlToFileOnce(url, destPath, abortSignal, source);
			return;
		} catch (error) {
			if (abortSignal?.aborted) throw error;
			if (isBusyError(error) || error instanceof MediaTransferBudgetError)
				throw error;

			const downloadError =
				error instanceof Error ? error : new Error(String(error));
			if (error instanceof MediaDownloadError && !error.retryable) {
				throw downloadError;
			}
			lastError = downloadError;
		}

		if (attempt < SEGMENT_DOWNLOAD_MAX_ATTEMPTS - 1) {
			await Bun.sleep(SEGMENT_DOWNLOAD_RETRY_BASE_MS * 2 ** attempt);
		}
	}

	throw lastError ?? new Error("Download failed");
}

// Storage that struggles with this many small concurrent reads can lower it through the env.
const SEGMENT_DOWNLOAD_CONCURRENCY = 64;

export function segmentDownloadConcurrencyFromEnv(
	env: Record<string, string | undefined> = process.env,
): number {
	const value = Number.parseInt(
		env.MEDIA_SERVER_SEGMENT_DOWNLOAD_CONCURRENCY ?? "",
		10,
	);
	return Number.isFinite(value) && value > 0
		? Math.min(value, SEGMENT_DOWNLOAD_CONCURRENCY)
		: SEGMENT_DOWNLOAD_CONCURRENCY;
}

interface SegmentTrackDownload {
	urls: string[];
	dir: string;
}

// Recordings are thousands of small segments, so latency per request dominates; video and audio
// share one pool so neither track waits for the other to finish.
export async function downloadSegmentTracks(
	tracks: SegmentTrackDownload[],
	jobId: string,
	progressBase: number,
	progressRange: number,
	abortSignal?: AbortSignal,
	sources?: Map<string, MuxSourceObject>,
): Promise<string[][]> {
	const { join } = await import("node:path");
	let completed = 0;
	let fatalError: Error | undefined;
	const outputPaths = tracks.map(
		(track) => new Array<string>(track.urls.length),
	);
	const pending = tracks.flatMap((track, trackIndex) => {
		const indexWidth = Math.max(3, String(track.urls.length).length);
		return track.urls.map((url, i) => ({
			trackIndex,
			i,
			url,
			outputPath: join(
				track.dir,
				`segment_${String(i + 1).padStart(indexWidth, "0")}.m4s`,
			),
		}));
	});
	const total = pending.length;
	let pendingIndex = 0;
	const batchAbortController = new AbortController();
	const batchSignal = abortSignal
		? AbortSignal.any([abortSignal, batchAbortController.signal])
		: batchAbortController.signal;

	async function worker() {
		while (pendingIndex < pending.length && !fatalError) {
			const entry = pending[pendingIndex++];
			if (!entry) break;
			const { trackIndex, i, url, outputPath } = entry;
			try {
				outputPaths[trackIndex][i] = outputPath;
				await downloadUrlToFile(
					url,
					outputPath,
					batchSignal,
					sources?.get(url),
				);
			} catch (err) {
				if (!fatalError) {
					fatalError = err instanceof Error ? err : new Error(String(err));
					pendingIndex = pending.length;
					batchAbortController.abort();
					console.error(
						`[mux-segments] Failed to download segment ${i + 1}/${tracks[trackIndex].urls.length}:`,
						err instanceof Error ? err.message : err,
					);
				}
				break;
			}
			completed++;
			if (total > 0) {
				updateJob(jobId, {
					phase: "downloading",
					progress:
						progressBase + Math.round((completed / total) * progressRange),
				});
			}
		}
	}

	await Promise.all(
		Array.from(
			{ length: Math.min(segmentDownloadConcurrencyFromEnv(), total) },
			() => worker(),
		),
	);
	if (fatalError) throw fatalError;
	return outputPaths;
}

function muxFailureErrorCode(
	error: unknown,
	stageErrorCode: RecordingErrorCode,
): RecordingErrorCode {
	if (error instanceof MediaTransferBudgetError)
		return "processing-budget-exhausted";
	if (error instanceof MediaDownloadError) return error.errorCode;
	if (
		isRetryableRecordingVerificationError(error) ||
		isBusyError(error) ||
		isTimeoutError(error)
	)
		return "processing-unavailable";
	return stageErrorCode;
}

function sendCurrentJobWebhook(jobId: string): void {
	const job = getJob(jobId);
	if (!job) {
		console.warn(`[mux-segments] Job ${jobId} not found for webhook`);
		return;
	}
	sendWebhook(job);
}

async function muxSegmentsAsync(
	jobId: string,
	videoId: string,
	outputUpload: StorageUploadTarget,
	thumbnailPresignedUrl: string | undefined,
	previewGifPresignedUrl: string | undefined,
	videoInitUrl: string,
	videoSegmentUrls: string[],
	audioInitUrl: string | null,
	audioSegmentUrls: string[] | null,
	outputVerificationUrl?: string,
	context: MuxContext = {},
): Promise<void> {
	const { ensureTempDir, getTempDir, retainTempPath } = await import(
		"../lib/temp-files"
	);
	const { lstat, mkdir, rm } = await import("node:fs/promises");
	const { join } = await import("node:path");

	const workDir = join(getTempDir(), `mux-${jobId}`);
	const releaseWorkDir = retainTempPath(workDir);
	const abortController = new AbortController();
	updateJob(jobId, { abortController });
	let outputUploadStarted = false;
	let pendingOutputUpload: PendingStorageUpload | undefined;
	let pendingOutputHash:
		| { promise: Promise<string>; abort: AbortController }
		| undefined;
	let processingTimeout: ReturnType<typeof setTimeout> | undefined;
	let errorCode: RecordingErrorCode = "processing-unavailable";
	const sources = context.sourceObjects
		? new Map(context.sourceObjects.map((source) => [source.url, source]))
		: undefined;
	const startedAt = Date.now();
	let previewAssets: Promise<PreviewAssets> | undefined;
	const previewAbort = new AbortController();

	try {
		if (!beginRecordingProcessing(jobId, SEGMENTED_RECORDING_TIMEOUT_MS))
			throw new Error("Recording processing job is no longer active");
		processingTimeout = setTimeout(() => {
			errorCode = "processing-unavailable";
			abortController.abort(new Error("Recording processing timed out"));
		}, SEGMENTED_RECORDING_TIMEOUT_MS);
		processingTimeout.unref?.();
		logVideoEvent("video_mux_started", {
			jobId,
			videoId,
			videoSegmentCount: videoSegmentUrls.length,
			audioSegmentCount: audioSegmentUrls?.length ?? 0,
			...getVideoCapacitySnapshot(),
		});
		await ensureTempDir();
		updateJob(jobId, { phase: "downloading", progress: 0 });
		sendCurrentJobWebhook(jobId);
		const webhookSecret = process.env.MEDIA_SERVER_WEBHOOK_SECRET;
		const preverifiedRuns =
			sources && webhookSecret && context.preverifiedRunUrls?.length
				? loadPreverifiedRuns(
						context.preverifiedRunUrls,
						webhookSecret,
						abortController.signal,
					).catch((): PreverifiedRunPayload[] => [])
				: Promise.resolve([]);

		await mkdir(workDir, { recursive: true });
		const videoDir = join(workDir, "video");
		const audioDir = join(workDir, "audio");
		await mkdir(videoDir, { recursive: true });
		await mkdir(audioDir, { recursive: true });

		const audioInput =
			audioInitUrl !== null &&
			audioSegmentUrls !== null &&
			audioSegmentUrls.length > 0
				? { initUrl: audioInitUrl, segmentUrls: audioSegmentUrls }
				: null;
		// Both init segments download together; a failure cancels the other and waits for it, so nothing
		// keeps writing into the work directory after the job has failed.
		const initAbort = new AbortController();
		const initSignal = AbortSignal.any([
			abortController.signal,
			initAbort.signal,
		]);
		const initDownloads = [
			downloadUrlToFile(
				videoInitUrl,
				join(videoDir, "init.mp4"),
				initSignal,
				sources?.get(videoInitUrl),
			),
			...(audioInput
				? [
						downloadUrlToFile(
							audioInput.initUrl,
							join(audioDir, "init.mp4"),
							initSignal,
							sources?.get(audioInput.initUrl),
						),
					]
				: []),
		];
		let initFailure: { error: unknown } | undefined;
		for (const download of initDownloads)
			download.catch((error: unknown) => {
				initFailure ??= { error };
				initAbort.abort();
			});
		await Promise.allSettled(initDownloads);
		if (initFailure) throw initFailure.error;
		updateJob(jobId, { phase: "downloading", progress: 5 });
		sendCurrentJobWebhook(jobId);

		const [videoSegmentFiles, audioSegmentFiles = []] =
			await downloadSegmentTracks(
				[
					{ urls: videoSegmentUrls, dir: videoDir },
					...(audioInput
						? [{ urls: audioInput.segmentUrls, dir: audioDir }]
						: []),
				],
				jobId,
				5,
				55,
				abortController.signal,
				sources,
			);

		updateJob(jobId, {
			phase: "processing",
			progress: 60,
			message: "Preparing recording tracks...",
		});
		sendCurrentJobWebhook(jobId);

		const combinedVideoPath = join(workDir, "combined_video.mp4");
		const videoInitPath = join(videoDir, "init.mp4");

		await withJobHeartbeat(jobId, () =>
			streamConcatFiles(
				[videoInitPath, ...videoSegmentFiles],
				combinedVideoPath,
				abortController.signal,
			),
		);
		const initSource = sources?.get(videoInitUrl);
		const segmentSources = videoSegmentUrls.map((url) => sources?.get(url));
		const preverified: PreverifiedVideoRun[] =
			initSource && segmentSources.every((source) => source !== undefined)
				? resolvePreverifiedRuns(
						await preverifiedRuns,
						initSource,
						segmentSources as MuxSourceObject[],
					)
				: [];
		// Parallel verification decodes runs of these fragments; otherwise free them now.
		const videoChunks =
			verifyDecodeChunks() > 1 || preverified.length > 0
				? { init: videoInitPath, segments: videoSegmentFiles }
				: undefined;
		if (!videoChunks) await rm(videoDir, { recursive: true, force: true });

		let combinedAudioPath: string | null = null;

		if (audioInput) {
			combinedAudioPath = join(workDir, "combined_audio.mp4");
			const audioInitPath = join(audioDir, "init.mp4");

			const audioOutputPath = combinedAudioPath;
			await withJobHeartbeat(jobId, () =>
				streamConcatFiles(
					[audioInitPath, ...audioSegmentFiles],
					audioOutputPath,
					abortController.signal,
				),
			);
			await rm(audioDir, { recursive: true, force: true });
		}

		if (hasCriticalMemoryPressure()) {
			throw new Error(VIDEO_MEMORY_PRESSURE_ERROR);
		}
		logVideoEvent("video_mux_inputs_materialized", {
			jobId,
			videoId,
			combinedVideoSize: file(combinedVideoPath).size,
			combinedAudioSize: combinedAudioPath ? file(combinedAudioPath).size : 0,
			resources: getSystemResources(),
		});

		const requiredAudio = context.requiredAudio ?? Boolean(audioInput);
		const resultPath = join(workDir, "result.mp4");
		errorCode = "output-invalid";
		updateJob(jobId, { progress: 65, message: "Combining video and audio..." });
		sendCurrentJobWebhook(jobId);
		await withJobHeartbeat(jobId, () =>
			withMuxMemoryGuard(abortController, () =>
				muxMediaTracksToMp4(
					combinedVideoPath,
					combinedAudioPath,
					resultPath,
					abortController.signal,
				),
			),
		).catch((error: unknown) => {
			if (error instanceof RecordingTimingError)
				errorCode = classifySourceError(error);
			throw error;
		});
		const beforeDecode = await lstat(resultPath, { bigint: true });
		if (!beforeDecode.isFile())
			throw new Error("Recording verification requires a local regular file");
		// Hashing and multipart parts only read the finished file, so they run during the decode check; the
		// upload is only completed (the object only exists) after everything below has passed.
		pendingOutputUpload = beginStorageUpload(
			resultPath,
			outputUpload,
			"video/mp4",
			abortController.signal,
		);
		const hashAbort = new AbortController();
		const outputHash = hashRecordingFile(
			resultPath,
			AbortSignal.any([abortController.signal, hashAbort.signal]),
		);
		outputHash.catch(() => {});
		pendingOutputHash = { promise: outputHash, abort: hashAbort };
		updateJob(jobId, {
			progress: 70,
			message: "Checking the processed recording...",
		});
		sendCurrentJobWebhook(jobId);
		const verificationStartedAt = Date.now();
		const localVerified = await withJobHeartbeat(jobId, () =>
			withMuxMemoryGuard(abortController, () =>
				verifyRemuxedRecording(
					combinedVideoPath,
					combinedAudioPath,
					resultPath,
					{
						requireAudio: requiredAudio,
						abortSignal: abortController.signal,
						onProgress: ({ frames, totalFrames }) => {
							updateJob(jobId, {
								progress: totalFrames
									? 70 + Math.min(4, Math.floor((5 * frames) / totalFrames))
									: 70,
								message: `Checking recording: ${frames.toLocaleString("en-US")} frames verified...`,
							});
						},
					},
					videoChunks,
					preverified,
				),
			),
		);
		if (videoChunks) await rm(videoDir, { recursive: true, force: true });
		logVideoEvent("video_mux_verification_complete", {
			jobId,
			videoId,
			durationMs: Date.now() - verificationStartedAt,
			preverifiedSegments: preverified.reduce(
				(total, run) => total + run.segments,
				0,
			),
			method: localVerified.integrity ? "decoded-source" : "encoded-packets",
			frames: localVerified.video.frameCount,
			recordingDuration: localVerified.video.duration,
			resources: getSystemResources(),
		});
		if (!localVerified.sourcePreserved)
			throw new Error("Recording source preservation was not verified");
		updateJob(jobId, {
			progress: 75,
			message: "Verifying the recording file...",
		});
		sendCurrentJobWebhook(jobId);
		const outputSha256 = await withJobHeartbeat(jobId, () => outputHash);
		const afterHash = await lstat(resultPath, { bigint: true });
		if (
			!afterHash.isFile() ||
			beforeDecode.dev !== afterHash.dev ||
			beforeDecode.ino !== afterHash.ino ||
			beforeDecode.size !== afterHash.size ||
			beforeDecode.mtimeNs !== afterHash.mtimeNs ||
			beforeDecode.ctimeNs !== afterHash.ctimeNs
		) {
			throw new Error("Local recording changed during verification");
		}
		await rm(combinedVideoPath, { force: true });
		if (combinedAudioPath) {
			await rm(combinedAudioPath, { force: true });
		}
		logVideoEvent("video_mux_output_ready", {
			jobId,
			videoId,
			outputSize: file(resultPath).size,
			durationMs: Date.now() - startedAt,
			resources: getSystemResources(),
		});
		const outputReadyAt = Date.now();
		// The result is final here, so render previews while the upload and its
		// remote check run; they are still only uploaded once the recording passes.
		previewAssets = renderPreviewAssets(
			resultPath,
			localVerified.video.duration,
			Boolean(thumbnailPresignedUrl),
			Boolean(previewGifPresignedUrl),
			AbortSignal.any([abortController.signal, previewAbort.signal]),
			videoId,
		);

		let metadata = await withJobHeartbeat(jobId, () =>
			probeVideoFile(resultPath),
		);
		if (
			!Number.isFinite(metadata.duration) ||
			metadata.duration <= 0 ||
			metadata.fileSize !== file(resultPath).size ||
			metadata.fileSize <= 0 ||
			metadata.width <= 0 ||
			metadata.height <= 0 ||
			!metadata.videoCodec ||
			(audioInput && !metadata.audioCodec)
		) {
			throw new Error(
				"Muxed recording is incomplete or missing a required track",
			);
		}
		metadata = { ...metadata, duration: localVerified.video.duration };

		updateJob(jobId, {
			phase: "uploading",
			progress: 80,
			message: "Uploading the processed recording...",
		});
		sendCurrentJobWebhook(jobId);

		outputUploadStarted = true;
		errorCode = "processing-unavailable";
		const startedOutputUpload = pendingOutputUpload;
		if (!startedOutputUpload)
			throw new Error("Recording output upload was not started");
		const uploadReceipt = await withJobHeartbeat(jobId, () =>
			startedOutputUpload.finish(),
		);
		if (outputVerificationUrl) {
			errorCode = "output-invalid";
			const uploadObjectIdentity = uploadReceipt.objectIdentity;
			if (!uploadObjectIdentity)
				throw new Error("Recording upload did not return an object identity");
			if (
				!beginRecordingVerification(
					jobId,
					PROCESS_TIMEOUT_MS + POST_VERIFICATION_ASSET_BUDGET_MS,
				)
			)
				throw new Error("Recording verification job is no longer active");
			updateJob(jobId, {
				progress: 90,
				message: "Verifying the uploaded recording...",
			});
			sendCurrentJobWebhook(jobId);
			const remoteBytes = await withJobHeartbeat(jobId, () =>
				verifyRemoteRecordingBytes(outputVerificationUrl, {
					expectedSha256: outputSha256,
					expectedFileSize: metadata.fileSize,
					expectedObjectIdentity: uploadObjectIdentity,
					abortSignal: abortController.signal,
				}),
			);
			const verified = { ...localVerified, ...remoteBytes };
			if (!verified.sourcePreserved || verified.remoteSha256 !== outputSha256)
				throw new Error(
					"Uploaded recording source preservation was not verified",
				);
			if (verified.fileSize !== metadata.fileSize)
				throw new Error(
					"Uploaded recording size does not match the muxed file",
				);
			metadata = {
				...metadata,
				duration: verified.video.duration,
				audioCodec: verified.audio ? (metadata.audioCodec ?? "unknown") : null,
				fileSize: verified.fileSize,
			};
			const manifestSha256 = getJob(jobId)?.manifestSha256;
			if (manifestSha256) {
				updateJob(jobId, {
					recordingVerification: {
						request: {
							version: 1,
							artifact: { kind: "segments", manifestSha256 },
							requiredAudio,
						},
						fullDecode: true,
						objectIdentity: verified.objectIdentity,
						...(context.outputKey && context.inventorySha256
							? {
									outputKey: context.outputKey,
									outputSha256: verified.remoteSha256,
									sourceProof: {
										version: 1 as const,
										manifestSha256,
										inventorySha256: context.inventorySha256,
										sourcePreserved: true as const,
										videoDuration: localVerified.video.duration,
										hasAudio: Boolean(localVerified.audio),
										audioVerified: Boolean(localVerified.audio),
									},
								}
							: {}),
					},
				});
			}
		}

		if (thumbnailPresignedUrl || previewGifPresignedUrl) {
			updateJob(jobId, {
				phase: "generating_thumbnail",
				progress: 95,
				message: "Generating preview assets...",
			});
			sendCurrentJobWebhook(jobId);
		}

		const uploadedAt = Date.now();
		await withJobHeartbeat(jobId, async () => {
			const assets = await previewAssets;
			abortController.signal.throwIfAborted();
			await Promise.all([
				thumbnailPresignedUrl && assets?.thumbnail
					? uploadToS3(
							assets.thumbnail,
							thumbnailPresignedUrl,
							"image/jpeg",
							abortController.signal,
						).catch((thumbErr) => {
							abortController.signal.throwIfAborted();
							console.warn(
								`[mux-segments] Thumbnail generation failed for ${videoId}:`,
								thumbErr,
							);
						})
					: undefined,
				previewGifPresignedUrl && assets?.previewGif
					? uploadFileToS3(
							assets.previewGif.path,
							previewGifPresignedUrl,
							"image/gif",
							abortController.signal,
						).catch((previewErr) => {
							abortController.signal.throwIfAborted();
							console.warn(
								"[mux-segments] Preview GIF generation failed:",
								previewErr,
							);
						})
					: undefined,
			]);
		});

		abortController.signal.throwIfAborted();
		updateJob(jobId, {
			phase: "complete",
			progress: 100,
			metadata,
		});
		if (!(context.audioLevels && context.outputKey))
			sendCurrentJobWebhook(jobId);
		logVideoEvent("video_mux_succeeded", {
			jobId,
			videoId,
			durationMs: Date.now() - startedAt,
			uploadMs: uploadedAt - outputReadyAt,
			previewMs: Date.now() - uploadedAt,
			metadata,
			resources: getSystemResources(),
		});
		if (context.audioLevels && context.outputKey) {
			if (processingTimeout) clearTimeout(processingTimeout);
			const completedJob = getJob(jobId);
			if (completedJob) {
				try {
					await sendWebhook(completedJob);
					await enhanceLocalRecording({
						path: resultPath,
						videoId,
						userId: completedJob.userId,
						jobId,
						sourceKey: context.outputKey,
						sourceIdentity: uploadReceipt.objectIdentity,
						duration: metadata.duration,
						webhookUrl: completedJob.webhookUrl,
						webhookSecret: completedJob.webhookSecret,
					});
				} catch {
					console.warn("[audio-levels] Original retained", { videoId });
				}
			}
		}
	} catch (error: unknown) {
		logVideoEvent("video_mux_failed", {
			jobId,
			videoId,
			durationMs: Date.now() - startedAt,
			errorCode: muxFailureErrorCode(error, errorCode),
			error: logReason(error, MUX_FAILURE_LOG_LENGTH),
			resources: getSystemResources(),
		});
		// Stop a hash still reading the output before the work directory is removed.
		pendingOutputHash?.abort.abort();
		await pendingOutputHash?.promise.catch(() => {});
		if (!outputUploadStarted) {
			// Parts still in flight could land after the abort, so let them settle first.
			await pendingOutputUpload?.cancel();
			await abortStorageUploadTarget(outputUpload).catch((abortError) => {
				console.warn(
					`[mux-segments] Failed to abort output upload for ${videoId}:`,
					abortError instanceof Error ? abortError.message : abortError,
				);
			});
		}
		console.error(`Mux-segments job ${jobId} failed:`, error);
		updateJob(jobId, {
			phase:
				abortController.signal.aborted && !isBusyError(error)
					? "cancelled"
					: "error",
			errorCode: muxFailureErrorCode(error, errorCode),
			error: isRetryableRecordingVerificationError(error)
				? "Recording verification temporarily unavailable (503)"
				: error instanceof Error
					? error.message
					: "Unknown error",
		});
		sendCurrentJobWebhook(jobId);
	} finally {
		if (processingTimeout) clearTimeout(processingTimeout);
		// Previews still rendering here belong to a job that failed before using them.
		previewAbort.abort();
		await (await previewAssets)?.previewGif?.cleanup();
		await rm(workDir, { recursive: true, force: true }).catch(() => {});
		releaseWorkDir();
	}
}

interface PreviewAssets {
	thumbnail: Uint8Array | null;
	previewGif: TempFileHandle | null;
}

// Never rejects: a failed preview only costs the asset, and the caller checks for abort.
async function renderPreviewAssets(
	resultPath: string,
	duration: number,
	thumbnail: boolean,
	previewGif: boolean,
	signal: AbortSignal,
	videoId: string,
): Promise<PreviewAssets> {
	const [thumbnailResult, previewGifResult] = await Promise.allSettled([
		thumbnail ? generateThumbnail(resultPath, duration, {}, signal) : null,
		previewGif ? generatePreviewGif(resultPath, duration, {}, signal) : null,
	]);
	if (thumbnailResult.status === "rejected" && !signal.aborted)
		console.warn(
			`[mux-segments] Thumbnail generation failed for ${videoId}:`,
			thumbnailResult.reason,
		);
	if (previewGifResult.status === "rejected" && !signal.aborted)
		console.warn(
			"[mux-segments] Preview GIF generation failed:",
			previewGifResult.reason,
		);
	return {
		thumbnail:
			thumbnailResult.status === "fulfilled" ? thumbnailResult.value : null,
		previewGif:
			previewGifResult.status === "fulfilled" ? previewGifResult.value : null,
	};
}

export default video;