Skip to content

Code-first video & image, rendered natively on WebGPU.

13.4 KB
Raw Blame
import type { RenderDiagnostic } from "@framefields/core";

/**
 * Quality checks run on a video while it renders: frames are measured as
 * they are read back for the encoder and audio as it is mixed, so the report
 * describes exactly what was encoded without decoding the file again.
 */
export interface VideoQaOptions {
	/** Luma (0-1) at or below which a pixel counts as black. Default 0.1. */
	readonly blackPixelThreshold?: number;
	/** Share of black pixels that makes a frame black. Default 0.98. */
	readonly blackFrameRatio?: number;
	/** Shortest run of black frames reported. Default 0.5 s. */
	readonly minBlackSec?: number;
	/** Shortest run of unchanging frames reported. Default 2 s. */
	readonly minFrozenSec?: number;
	/** Level (dBFS, 100 ms windows) below which audio counts as silent. Default -60. */
	readonly silenceDbfs?: number;
	/** Shortest silent stretch reported. Default 2 s. */
	readonly minSilenceSec?: number;
	/** Integrated loudness to hit, e.g. -14 for most streaming platforms. Unchecked when unset. */
	readonly targetLufs?: number;
	/** Allowed distance from `targetLufs`, in LU. Default 1. */
	readonly lufsTolerance?: number;
}

export interface QaSegment {
	readonly startSec: number;
	readonly endSec: number;
}

export interface QaIssue {
	readonly severity: "error" | "warning" | "info";
	/** Stable machine-readable kind, e.g. "black_frames", "audio_clipping" */
	readonly code: string;
	readonly message: string;
	readonly startSec?: number;
	readonly endSec?: number;
	readonly layerId?: string;
}

export interface VideoQaStats {
	readonly expectedFrames: number;
	readonly encodedFrames: number;
	readonly fps: number;
	readonly width: number;
	readonly height: number;
	/** Average luma over the whole video, 0-1 */
	readonly meanLuma: number;
	readonly blackSegments: readonly QaSegment[];
	readonly frozenSegments: readonly QaSegment[];
}

export interface AudioQaStats {
	readonly sampleRate: number;
	readonly channels: number;
	readonly durationSec: number;
	/** Highest sample, dBFS (sample peak, not true peak) */
	readonly peakDbfs: number;
	/** Mean level over the whole track, dBFS */
	readonly rmsDbfs: number;
	/** ITU-R BS.1770 integrated loudness; null when the track is silent */
	readonly integratedLufs: number | null;
	/** Samples at or beyond full scale */
	readonly clippedSamples: number;
	readonly silentSegments: readonly QaSegment[];
}

export interface VideoQaReport {
	/** No errors or warnings (info issues do not fail a render) */
	readonly passed: boolean;
	readonly video?: VideoQaStats;
	readonly audio?: AudioQaStats;
	readonly issues: readonly QaIssue[];
}

const DEFAULTS = {
	blackPixelThreshold: 0.1,
	blackFrameRatio: 0.98,
	minBlackSec: 0.5,
	minFrozenSec: 2,
	silenceDbfs: -60,
	minSilenceSec: 2,
	lufsTolerance: 1,
} as const;

// Every 4th pixel of every 4th row: 1/16 of the frame keeps inspection
// around a millisecond at 1080p while still seeing anything frame-sized.
const SAMPLE_STEP = 4;
// Mean absolute luma change (0-255) under which two frames are the same picture.
const FROZEN_DIFF = 0.25;

/** Measures frames handed to it in order, as RGBA bytes. */
export class FrameInspector {
	private readonly blackLuma: number;
	private readonly blackRatio: number;
	private readonly black: boolean[] = [];
	private readonly sameAsPrevious: boolean[] = [];
	private lumaSum = 0;
	private previous: Uint8Array | null = null;

	constructor(options: VideoQaOptions = {}) {
		this.blackLuma =
			(options.blackPixelThreshold ?? DEFAULTS.blackPixelThreshold) * 255;
		this.blackRatio = options.blackFrameRatio ?? DEFAULTS.blackFrameRatio;
	}

	get frameCount(): number {
		return this.black.length;
	}

	inspect(rgba: Uint8Array, width: number, height: number): void {
		const cols = Math.ceil(width / SAMPLE_STEP);
		const rows = Math.ceil(height / SAMPLE_STEP);
		const luma = new Uint8Array(cols * rows);
		let blackCount = 0;
		let sum = 0;
		let k = 0;
		for (let y = 0; y < height; y += SAMPLE_STEP) {
			let p = y * width * 4;
			for (let x = 0; x < width; x += SAMPLE_STEP, p += SAMPLE_STEP * 4) {
				// BT.709 luma from 8-bit RGB
				const l =
					0.2126 * rgba[p]! + 0.7152 * rgba[p + 1]! + 0.0722 * rgba[p + 2]!;
				luma[k++] = l;
				sum += l;
				if (l <= this.blackLuma) blackCount++;
			}
		}
		const n = luma.length;
		this.lumaSum += sum / n / 255;
		this.black.push(blackCount / n >= this.blackRatio);

		let same = false;
		if (this.previous && this.previous.length === n) {
			let diff = 0;
			for (let i = 0; i < n; i++)
				diff += Math.abs(luma[i]! - this.previous[i]!);
			same = diff / n < FROZEN_DIFF;
		}
		this.sameAsPrevious.push(same);
		this.previous = luma;
	}

	stats(
		expectedFrames: number,
		encodedFrames: number,
		fps: number,
		width: number,
		height: number,
		options: VideoQaOptions = {},
	): VideoQaStats {
		const minBlack = Math.max(
			1,
			Math.round((options.minBlackSec ?? DEFAULTS.minBlackSec) * fps),
		);
		const minFrozen = Math.max(
			2,
			Math.round((options.minFrozenSec ?? DEFAULTS.minFrozenSec) * fps),
		);
		const blackSegments = runs(this.black, minBlack).map(([a, b]) => ({
			startSec: a / fps,
			endSec: b / fps,
		}));
		// A frozen run starts on the frame before its first repeat.
		const frozenSegments = runs(this.sameAsPrevious, minFrozen - 1).map(
			([a, b]) => ({ startSec: (a - 1) / fps, endSec: b / fps }),
		);
		return {
			expectedFrames,
			encodedFrames,
			fps,
			width,
			height,
			meanLuma: this.frameCount > 0 ? this.lumaSum / this.frameCount : 0,
			blackSegments,
			frozenSegments,
		};
	}
}

/** [start, end) index ranges where flags hold for at least minLength in a row. */
function runs(
	flags: readonly boolean[],
	minLength: number,
): [number, number][] {
	const out: [number, number][] = [];
	let start = -1;
	for (let i = 0; i <= flags.length; i++) {
		if (i < flags.length && flags[i]) {
			if (start < 0) start = i;
		} else if (start >= 0) {
			if (i - start >= minLength) out.push([start, i]);
			start = -1;
		}
	}
	return out;
}

const toDb = (power: number) =>
	power > 0 ? 10 * Math.log10(power) : -Infinity;

/** Measures a mixed audio track (one Float32Array per channel). */
export function analyzeAudio(
	channels: readonly Float32Array[],
	sampleRate: number,
	options: VideoQaOptions = {},
): AudioQaStats {
	const length = channels[0]?.length ?? 0;
	let peak = 0;
	let sumSquares = 0;
	let clipped = 0;
	for (const ch of channels) {
		for (let i = 0; i < length; i++) {
			const v = Math.abs(ch[i]!);
			if (v > peak) peak = v;
			if (v >= 0.9999) clipped++;
			sumSquares += v * v;
		}
	}
	const total = length * Math.max(1, channels.length);

	const silenceDbfs = options.silenceDbfs ?? DEFAULTS.silenceDbfs;
	const window = Math.max(1, Math.round(sampleRate * 0.1));
	const silent: boolean[] = [];
	for (let start = 0; start < length; start += window) {
		const end = Math.min(length, start + window);
		let s = 0;
		for (const ch of channels)
			for (let i = start; i < end; i++) s += ch[i]! * ch[i]!;
		silent.push(toDb(s / ((end - start) * channels.length)) < silenceDbfs);
	}
	const minSilence = Math.max(
		1,
		Math.round((options.minSilenceSec ?? DEFAULTS.minSilenceSec) / 0.1),
	);
	const silentSegments = runs(silent, minSilence).map(([a, b]) => ({
		startSec: (a * window) / sampleRate,
		endSec: Math.min(length, b * window) / sampleRate,
	}));

	const peakDbfs = 20 * Math.log10(Math.max(peak, 1e-12));
	return {
		sampleRate,
		channels: channels.length,
		durationSec: length / sampleRate,
		peakDbfs,
		rmsDbfs: toDb(sumSquares / Math.max(1, total)),
		integratedLufs:
			peakDbfs < silenceDbfs ? null : integratedLoudness(channels, sampleRate),
		clippedSamples: clipped,
		silentSegments,
	};
}

interface Biquad {
	b0: number;
	b1: number;
	b2: number;
	a1: number;
	a2: number;
}

/** BS.1770 K-weighting stage 1: +4 dB high shelf around 1.5 kHz (the head). */
function highShelf(sampleRate: number): Biquad {
	const A = 10 ** (4 / 40);
	const w0 = (2 * Math.PI * 1500) / sampleRate;
	const cos = Math.cos(w0);
	const alpha = Math.sin(w0) / (2 / Math.SQRT2);
	const sq = 2 * Math.sqrt(A) * alpha;
	const a0 = A + 1 - (A - 1) * cos + sq;
	return {
		b0: (A * (A + 1 + (A - 1) * cos + sq)) / a0,
		b1: (-2 * A * (A - 1 + (A + 1) * cos)) / a0,
		b2: (A * (A + 1 + (A - 1) * cos - sq)) / a0,
		a1: (2 * (A - 1 - (A + 1) * cos)) / a0,
		a2: (A + 1 - (A - 1) * cos - sq) / a0,
	};
}

/** BS.1770 K-weighting stage 2: 38 Hz high pass, Q 0.5 (the RLB curve). */
function highPass(sampleRate: number): Biquad {
	const w0 = (2 * Math.PI * 38) / sampleRate;
	const cos = Math.cos(w0);
	const alpha = Math.sin(w0) / (2 * 0.5);
	const a0 = 1 + alpha;
	return {
		b0: (1 + cos) / 2 / a0,
		b1: -(1 + cos) / a0,
		b2: (1 + cos) / 2 / a0,
		a1: (-2 * cos) / a0,
		a2: (1 - alpha) / a0,
	};
}

function filter(input: Float32Array, f: Biquad): Float32Array {
	const out = new Float32Array(input.length);
	let x1 = 0;
	let x2 = 0;
	let y1 = 0;
	let y2 = 0;
	for (let i = 0; i < input.length; i++) {
		const x = input[i]!;
		const y = f.b0 * x + f.b1 * x1 + f.b2 * x2 - f.a1 * y1 - f.a2 * y2;
		out[i] = y;
		x2 = x1;
		x1 = x;
		y2 = y1;
		y1 = y;
	}
	return out;
}

/**
 * ITU-R BS.1770-4 integrated loudness (LUFS): K-weighted mean square in
 * 400 ms blocks with 75% overlap, gated at -70 LUFS and then at 10 LU below
 * the mean of what passed. Channels weigh 1 (stereo and mono layouts).
 */
export function integratedLoudness(
	channels: readonly Float32Array[],
	sampleRate: number,
): number | null {
	const shelf = highShelf(sampleRate);
	const hp = highPass(sampleRate);
	const weighted = channels.map((ch) => filter(filter(ch, shelf), hp));
	const length = weighted[0]?.length ?? 0;
	const block = Math.round(0.4 * sampleRate);
	const hop = Math.round(0.1 * sampleRate);
	if (length < block) return null;

	const blockPower: number[] = [];
	for (let start = 0; start + block <= length; start += hop) {
		let power = 0;
		for (const ch of weighted) {
			let s = 0;
			for (let i = start; i < start + block; i++) s += ch[i]! * ch[i]!;
			power += s / block;
		}
		blockPower.push(power);
	}
	const lufs = (power: number) => -0.691 + toDb(power);
	const mean = (xs: number[]) => xs.reduce((a, b) => a + b, 0) / xs.length;

	const absolute = blockPower.filter((p) => lufs(p) > -70);
	if (absolute.length === 0) return null;
	const relativeGate = lufs(mean(absolute)) - 10;
	const gated = absolute.filter((p) => lufs(p) > relativeGate);
	return gated.length > 0 ? lufs(mean(gated)) : null;
}

/** Turn measurements and document diagnostics into a report with issues. */
export function buildQaReport(parts: {
	video?: VideoQaStats;
	audio?: AudioQaStats;
	diagnostics?: readonly RenderDiagnostic[];
	options?: VideoQaOptions;
}): VideoQaReport {
	const { video, audio, options = {} } = parts;
	const issues: QaIssue[] = [];
	const at = (s: QaSegment) =>
		`${s.startSec.toFixed(2)}–${s.endSec.toFixed(2)} s`;

	if (video) {
		if (video.encodedFrames !== video.expectedFrames) {
			issues.push({
				severity: "error",
				code: "frames_missing",
				message: `${video.encodedFrames} of ${video.expectedFrames} frames were encoded`,
			});
		}
		for (const s of video.blackSegments) {
			issues.push({
				severity: "warning",
				code: "black_frames",
				message: `Black frames at ${at(s)}`,
				...s,
			});
		}
		for (const s of video.frozenSegments) {
			issues.push({
				severity: "info",
				code: "frozen_video",
				message: `Picture does not change at ${at(s)}`,
				...s,
			});
		}
	}

	if (audio) {
		if (audio.integratedLufs === null) {
			issues.push({
				severity: "info",
				code: "audio_silent",
				message: "The audio track is silent",
			});
		} else {
			if (audio.clippedSamples > 0) {
				issues.push({
					severity: "warning",
					code: "audio_clipping",
					message: `${audio.clippedSamples} audio sample(s) at or beyond full scale (peak ${audio.peakDbfs.toFixed(1)} dBFS)`,
				});
			}
			for (const s of audio.silentSegments) {
				issues.push({
					severity: "info",
					code: "audio_silence",
					message: `Silence at ${at(s)}`,
					...s,
				});
			}
			if (options.targetLufs !== undefined) {
				const off = audio.integratedLufs - options.targetLufs;
				const tolerance = options.lufsTolerance ?? DEFAULTS.lufsTolerance;
				if (Math.abs(off) > tolerance) {
					issues.push({
						severity: "warning",
						code: "loudness_off_target",
						message: `Integrated loudness ${audio.integratedLufs.toFixed(1)} LUFS is ${Math.abs(off).toFixed(1)} LU ${off > 0 ? "above" : "below"} the ${options.targetLufs} LUFS target`,
					});
				}
			}
		}
	}

	for (const d of parts.diagnostics ?? []) {
		issues.push({
			severity: d.severity,
			code: d.code,
			message: d.message,
			...(d.layerId !== undefined && { layerId: d.layerId }),
		});
	}

	return {
		passed: !issues.some((i) => i.severity !== "info"),
		...(video && { video }),
		...(audio && { audio }),
		issues,
	};
}

/** A short human- and agent-readable summary of a report. */
export function formatQaReport(report: VideoQaReport): string {
	const lines = [`QA ${report.passed ? "passed" : "FAILED"}`];
	const v = report.video;
	if (v) {
		lines.push(
			`  video: ${v.encodedFrames}/${v.expectedFrames} frames, ${v.width}x${v.height} @ ${v.fps} fps, mean luma ${v.meanLuma.toFixed(2)}`,
		);
	}
	const a = report.audio;
	if (a) {
		const lufs =
			a.integratedLufs === null
				? "silent"
				: `${a.integratedLufs.toFixed(1)} LUFS`;
		lines.push(
			`  audio: ${a.durationSec.toFixed(2)} s, ${lufs}, peak ${a.peakDbfs.toFixed(1)} dBFS, rms ${a.rmsDbfs.toFixed(1)} dBFS`,
		);
	}
	for (const i of report.issues) {
		lines.push(`  ${i.severity.padEnd(7)} ${i.code}: ${i.message}`);
	}
	return lines.join("\n");
}