LLM inference in C/C++
C++ ggml
ui: Add HEIC/HEIF image support (#24137)
Nicolas Mowen5f04dc7ac3f4
diff --git a/tools/ui/src/lib/constants/image-size.ts b/tools/ui/src/lib/constants/image-size.tsindex bcc1da08b1..8a7f921fa0 100644--- a/tools/ui/src/lib/constants/image-size.ts+++ b/tools/ui/src/lib/constants/image-size.ts@@ -1 +1,3 @@export const MEGAPIXELS_TO_PIXELS = 1_000_000;++export const HEIC_JPEG_QUALITY = 0.85;diff --git a/tools/ui/src/lib/constants/supported-file-types.ts b/tools/ui/src/lib/constants/supported-file-types.tsindex 4141161548..cbe780aa57 100644--- a/tools/ui/src/lib/constants/supported-file-types.ts+++ b/tools/ui/src/lib/constants/supported-file-types.ts@@ -63,6 +63,10 @@ export const IMAGE_FILE_TYPES = {[FileTypeImage.SVG]: {extensions: [FileExtensionImage.SVG],mimeTypes: [MimeTypeImage.SVG]+ },+ [FileTypeImage.HEIC]: {+ extensions: [FileExtensionImage.HEIC, FileExtensionImage.HEIF],+ mimeTypes: [MimeTypeImage.HEIC, MimeTypeImage.HEIF]}} as const;diff --git a/tools/ui/src/lib/enums/files.enums.ts b/tools/ui/src/lib/enums/files.enums.tsindex 8008a1040b..dbe07ea586 100644--- a/tools/ui/src/lib/enums/files.enums.ts+++ b/tools/ui/src/lib/enums/files.enums.ts@@ -25,7 +25,9 @@ export enum FileTypeImage {PNG = 'png',GIF = 'gif',WEBP = 'webp',- SVG = 'svg'+ SVG = 'svg',+ HEIC = 'heic',+ HEIF = 'heif'}export enum FileTypeAudio {@@ -90,7 +92,9 @@ export enum FileExtensionImage {PNG = '.png',GIF = '.gif',WEBP = '.webp',- SVG = '.svg'+ SVG = '.svg',+ HEIC = '.heic',+ HEIF = '.heif'}export enum FileExtensionAudio {@@ -205,7 +209,9 @@ export enum MimeTypeImage {WEBP = 'image/webp',SVG = 'image/svg+xml',ICO = 'image/x-icon',- ICO_MICROSOFT = 'image/vnd.microsoft.icon'+ ICO_MICROSOFT = 'image/vnd.microsoft.icon',+ HEIC = 'image/heic',+ HEIF = 'image/heif'}export enum MimeTypeText {diff --git a/tools/ui/src/lib/utils/file-type.ts b/tools/ui/src/lib/utils/file-type.tsindex d14efbc350..e61564174e 100644--- a/tools/ui/src/lib/utils/file-type.ts+++ b/tools/ui/src/lib/utils/file-type.ts@@ -30,6 +30,8 @@ export function getFileTypeCategory(mimeType: string): FileTypeCategory | null {case MimeTypeImage.GIF:case MimeTypeImage.WEBP:case MimeTypeImage.SVG:+ case MimeTypeImage.HEIC:+ case MimeTypeImage.HEIF:return FileTypeCategory.IMAGE;// Audio@@ -118,6 +120,8 @@ export function getFileTypeCategoryByExtension(filename: string): FileTypeCategocase FileExtensionImage.GIF:case FileExtensionImage.WEBP:case FileExtensionImage.SVG:+ case FileExtensionImage.HEIC:+ case FileExtensionImage.HEIF:return FileTypeCategory.IMAGE;// Audiodiff --git a/tools/ui/src/lib/utils/heic-to-jpeg.ts b/tools/ui/src/lib/utils/heic-to-jpeg.tsnew file mode 100644index 0000000000..4ccb992a3b--- /dev/null+++ b/tools/ui/src/lib/utils/heic-to-jpeg.ts@@ -0,0 +1,56 @@+import { MimeTypeImage } from '$lib/enums';+import { HEIC_JPEG_QUALITY } from '$lib/constants/image-size';++// heic requires a relatively large decoder, in order to reduce primary bundle size+// we lazily load this decoder from a CDN when needed, and cache it for future conversions+const HEIC_TO_CDN_URL = 'https://cdn.jsdelivr.net/npm/[email protected]/dist/heic-to.js';++interface HeicToModule {+ heicTo(args: { blob: Blob; type: string; quality?: number }): Promise<Blob>;+}++let modulePromise: Promise<HeicToModule> | null = null;++/**+ * Lazily load the heic-to decoder from the CDN and cache it+ * @returns Promise resolving to the heic-to module+ */+function getHeicTo(): Promise<HeicToModule> {+ if (!modulePromise) {+ modulePromise = import(/* @vite-ignore */ HEIC_TO_CDN_URL) as Promise<HeicToModule>;+ }++ return modulePromise;+}++/**+ * Convert a HEIC/HEIF file to a compressed JPEG data URL+ * @param file - The HEIC/HEIF file to convert+ * @returns Promise resolving to JPEG data URL+ */+export async function heicFileToJpegDataURL(file: File | Blob): Promise<string> {+ const { heicTo } = await getHeicTo();+ const jpegBlob = await heicTo({+ blob: file,+ type: MimeTypeImage.JPEG,+ quality: HEIC_JPEG_QUALITY+ });++ return new Promise((resolve, reject) => {+ const reader = new FileReader();+ reader.onload = () => resolve(reader.result as string);+ reader.onerror = () => reject(reader.error);+ reader.readAsDataURL(jpegBlob);+ });+}++/**+ * Check if a MIME type represents a HEIC/HEIF image+ * @param mimeType - The MIME type to check+ * @returns True if the MIME type is image/heic or image/heif+ */+export function isHeicMimeType(mimeType: string): boolean {+ const normalized = mimeType.trim().toLowerCase();++ return normalized === MimeTypeImage.HEIC || normalized === MimeTypeImage.HEIF;+}diff --git a/tools/ui/src/lib/utils/process-uploaded-files.ts b/tools/ui/src/lib/utils/process-uploaded-files.tsindex d0110f6cbc..91b619cfb9 100644--- a/tools/ui/src/lib/utils/process-uploaded-files.ts+++ b/tools/ui/src/lib/utils/process-uploaded-files.ts@@ -1,5 +1,6 @@import { isSvgMimeType, svgBase64UrlToPngDataURL } from './svg-to-png';import { isWebpMimeType, webpBase64UrlToPngDataURL } from './webp-to-png';+import { heicFileToJpegDataURL, isHeicMimeType } from './heic-to-jpeg';import { FileTypeCategory } from '$lib/enums';import { SETTINGS_KEYS } from '$lib/constants';import { modelsStore } from '$lib/stores/models.svelte';@@ -68,7 +69,7 @@ export async function processFilesToChatUploaded(if (getFileTypeCategory(file.type) === FileTypeCategory.IMAGE) {let preview = await readFileAsDataURL(file);- // Normalize SVG and WebP to PNG in previews+ // Normalize SVG and WebP to PNG, and HEIC to compressed JPEG, in previewsif (isSvgMimeType(file.type)) {try {preview = await svgBase64UrlToPngDataURL(preview);@@ -81,6 +82,13 @@ export async function processFilesToChatUploaded(} catch (err) {console.error('Failed to convert WebP to PNG:', err);}+ } else if (isHeicMimeType(file.type)) {+ try {+ preview = await heicFileToJpegDataURL(file);+ } catch (err) {+ console.error('Failed to convert HEIC to PNG:', err);+ continue;+ }}results.push({ ...base, preview });