Skip to content

🎥 Make videos programmatically with React

3.7 KB
Raw Blame
export const WHISPER_WEBGPU_MODELS = [
	'tiny',
	'tiny.en',
	'base',
	'base.en',
	'small',
	'small.en',
	'medium',
	'medium.en',
	'large-v3-turbo',
] as const;

export type WhisperWebGpuModel = (typeof WHISPER_WEBGPU_MODELS)[number];

export type WhisperWebGpuModelInfo = {
	name: WhisperWebGpuModel;
	modelId: string;
	parameters: number;
	multilingual: boolean;
	supportsTranslation: boolean;
	webGpuDownloadSize: number;
};

const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {
	tiny: {
		name: 'tiny',
		modelId: 'onnx-community/whisper-tiny_timestamped',
		parameters: 39_000_000,
		multilingual: true,
		supportsTranslation: true,
		webGpuDownloadSize: 119_699_015,
	},
	'tiny.en': {
		name: 'tiny.en',
		modelId: 'onnx-community/whisper-tiny.en_timestamped',
		parameters: 39_000_000,
		multilingual: false,
		supportsTranslation: false,
		webGpuDownloadSize: 119_697_479,
	},
	base: {
		name: 'base',
		modelId: 'onnx-community/whisper-base_timestamped',
		parameters: 74_000_000,
		multilingual: true,
		supportsTranslation: true,
		webGpuDownloadSize: 206_190_057,
	},
	'base.en': {
		name: 'base.en',
		modelId: 'onnx-community/whisper-base.en_timestamped',
		parameters: 74_000_000,
		multilingual: false,
		supportsTranslation: false,
		webGpuDownloadSize: 206_188_009,
	},
	small: {
		name: 'small',
		modelId: 'onnx-community/whisper-small_timestamped',
		parameters: 244_000_000,
		multilingual: true,
		supportsTranslation: true,
		webGpuDownloadSize: 586_213_010,
	},
	'small.en': {
		name: 'small.en',
		modelId: 'onnx-community/whisper-small.en_timestamped',
		parameters: 244_000_000,
		multilingual: false,
		supportsTranslation: false,
		webGpuDownloadSize: 586_209_938,
	},
	medium: {
		name: 'medium',
		modelId: 'onnx-community/whisper-medium_timestamped',
		parameters: 769_000_000,
		multilingual: true,
		supportsTranslation: true,
		webGpuDownloadSize: 1_698_508_143,
	},
	'medium.en': {
		name: 'medium.en',
		modelId: 'onnx-community/whisper-medium.en_timestamped',
		parameters: 769_000_000,
		multilingual: false,
		supportsTranslation: false,
		webGpuDownloadSize: 1_698_504_047,
	},
	'large-v3-turbo': {
		name: 'large-v3-turbo',
		modelId: 'onnx-community/whisper-large-v3-turbo_timestamped',
		parameters: 809_000_000,
		multilingual: true,
		supportsTranslation: false,
		webGpuDownloadSize: 1_608_611_679,
	},
};

const HOSTED_MODEL_IDS: Record<WhisperWebGpuModel, string> = {
	tiny: 'whisper-tiny_timestamped-v1',
	'tiny.en': 'whisper-tiny.en_timestamped-v1',
	base: 'whisper-base_timestamped-v1',
	'base.en': 'whisper-base.en_timestamped-v1',
	small: 'whisper-small_timestamped-v1',
	'small.en': 'whisper-small.en_timestamped-v1',
	medium: 'whisper-medium_timestamped-v1',
	'medium.en': 'whisper-medium.en_timestamped-v1',
	'large-v3-turbo': 'whisper-large-v3-turbo_timestamped-v1',
};

export const getAvailableModels = (): WhisperWebGpuModelInfo[] => {
	return WHISPER_WEBGPU_MODELS.map((model) => ({...MODEL_INFO[model]}));
};

export const getModelInfo = (
	model: WhisperWebGpuModel,
): WhisperWebGpuModelInfo => {
	return MODEL_INFO[model];
};

export const getHostedModelId = (model: WhisperWebGpuModel): string => {
	return HOSTED_MODEL_IDS[model];
};

const DEFAULT_WHISPER_WEBGPU_DTYPE = {
	encoder_model: 'fp32',
	decoder_model_merged: 'q4',
} as const;

// The FP32 encoder is 2.55 GB and Transformers.js reads it into one ArrayBuffer,
// which exceeds the browser allocation limit.
const LARGE_V3_TURBO_WHISPER_WEBGPU_DTYPE = {
	encoder_model: 'fp16',
	decoder_model_merged: 'q4',
} as const;

export const getWhisperWebGpuDtype = (model: WhisperWebGpuModel) => {
	return model === 'large-v3-turbo'
		? LARGE_V3_TURBO_WHISPER_WEBGPU_DTYPE
		: DEFAULT_WHISPER_WEBGPU_DTYPE;
};