Skip to content

LLM inference in C/C++

C++ ggml

ui: Add HEIC/HEIF image support (#24137)

Nicolas Mowen5f04dc7ac3f4
diff --git a/tools/ui/src/lib/constants/image-size.ts b/tools/ui/src/lib/constants/image-size.ts
index bcc1da08b1..8a7f921fa0 100644
--- a/tools/ui/src/lib/constants/image-size.ts
+++ b/tools/ui/src/lib/constants/image-size.ts
@@ -1 +1,3 @@
export const MEGAPIXELS_TO_PIXELS = 1_000_000;
+
+export const HEIC_JPEG_QUALITY = 0.85;
diff --git a/tools/ui/src/lib/constants/supported-file-types.ts b/tools/ui/src/lib/constants/supported-file-types.ts
index 4141161548..cbe780aa57 100644
--- a/tools/ui/src/lib/constants/supported-file-types.ts
+++ b/tools/ui/src/lib/constants/supported-file-types.ts
@@ -63,6 +63,10 @@ export const IMAGE_FILE_TYPES = {
[FileTypeImage.SVG]: {
extensions: [FileExtensionImage.SVG],
mimeTypes: [MimeTypeImage.SVG]
+ },
+ [FileTypeImage.HEIC]: {
+ extensions: [FileExtensionImage.HEIC, FileExtensionImage.HEIF],
+ mimeTypes: [MimeTypeImage.HEIC, MimeTypeImage.HEIF]
}
} as const;
diff --git a/tools/ui/src/lib/enums/files.enums.ts b/tools/ui/src/lib/enums/files.enums.ts
index 8008a1040b..dbe07ea586 100644
--- a/tools/ui/src/lib/enums/files.enums.ts
+++ b/tools/ui/src/lib/enums/files.enums.ts
@@ -25,7 +25,9 @@ export enum FileTypeImage {
PNG = 'png',
GIF = 'gif',
WEBP = 'webp',
- SVG = 'svg'
+ SVG = 'svg',
+ HEIC = 'heic',
+ HEIF = 'heif'
}
export enum FileTypeAudio {
@@ -90,7 +92,9 @@ export enum FileExtensionImage {
PNG = '.png',
GIF = '.gif',
WEBP = '.webp',
- SVG = '.svg'
+ SVG = '.svg',
+ HEIC = '.heic',
+ HEIF = '.heif'
}
export enum FileExtensionAudio {
@@ -205,7 +209,9 @@ export enum MimeTypeImage {
WEBP = 'image/webp',
SVG = 'image/svg+xml',
ICO = 'image/x-icon',
- ICO_MICROSOFT = 'image/vnd.microsoft.icon'
+ ICO_MICROSOFT = 'image/vnd.microsoft.icon',
+ HEIC = 'image/heic',
+ HEIF = 'image/heif'
}
export enum MimeTypeText {
diff --git a/tools/ui/src/lib/utils/file-type.ts b/tools/ui/src/lib/utils/file-type.ts
index d14efbc350..e61564174e 100644
--- a/tools/ui/src/lib/utils/file-type.ts
+++ b/tools/ui/src/lib/utils/file-type.ts
@@ -30,6 +30,8 @@ export function getFileTypeCategory(mimeType: string): FileTypeCategory | null {
case MimeTypeImage.GIF:
case MimeTypeImage.WEBP:
case MimeTypeImage.SVG:
+ case MimeTypeImage.HEIC:
+ case MimeTypeImage.HEIF:
return FileTypeCategory.IMAGE;
// Audio
@@ -118,6 +120,8 @@ export function getFileTypeCategoryByExtension(filename: string): FileTypeCatego
case FileExtensionImage.GIF:
case FileExtensionImage.WEBP:
case FileExtensionImage.SVG:
+ case FileExtensionImage.HEIC:
+ case FileExtensionImage.HEIF:
return FileTypeCategory.IMAGE;
// Audio
diff --git a/tools/ui/src/lib/utils/heic-to-jpeg.ts b/tools/ui/src/lib/utils/heic-to-jpeg.ts
new file mode 100644
index 0000000000..4ccb992a3b
--- /dev/null
+++ b/tools/ui/src/lib/utils/heic-to-jpeg.ts
@@ -0,0 +1,56 @@
+import { MimeTypeImage } from '$lib/enums';
+import { HEIC_JPEG_QUALITY } from '$lib/constants/image-size';
+
+// heic requires a relatively large decoder, in order to reduce primary bundle size
+// we lazily load this decoder from a CDN when needed, and cache it for future conversions
+const HEIC_TO_CDN_URL = 'https://cdn.jsdelivr.net/npm/[email protected]/dist/heic-to.js';
+
+interface HeicToModule {
+ heicTo(args: { blob: Blob; type: string; quality?: number }): Promise<Blob>;
+}
+
+let modulePromise: Promise<HeicToModule> | null = null;
+
+/**
+ * Lazily load the heic-to decoder from the CDN and cache it
+ * @returns Promise resolving to the heic-to module
+ */
+function getHeicTo(): Promise<HeicToModule> {
+ if (!modulePromise) {
+ modulePromise = import(/* @vite-ignore */ HEIC_TO_CDN_URL) as Promise<HeicToModule>;
+ }
+
+ return modulePromise;
+}
+
+/**
+ * Convert a HEIC/HEIF file to a compressed JPEG data URL
+ * @param file - The HEIC/HEIF file to convert
+ * @returns Promise resolving to JPEG data URL
+ */
+export async function heicFileToJpegDataURL(file: File | Blob): Promise<string> {
+ const { heicTo } = await getHeicTo();
+ const jpegBlob = await heicTo({
+ blob: file,
+ type: MimeTypeImage.JPEG,
+ quality: HEIC_JPEG_QUALITY
+ });
+
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = () => resolve(reader.result as string);
+ reader.onerror = () => reject(reader.error);
+ reader.readAsDataURL(jpegBlob);
+ });
+}
+
+/**
+ * Check if a MIME type represents a HEIC/HEIF image
+ * @param mimeType - The MIME type to check
+ * @returns True if the MIME type is image/heic or image/heif
+ */
+export function isHeicMimeType(mimeType: string): boolean {
+ const normalized = mimeType.trim().toLowerCase();
+
+ return normalized === MimeTypeImage.HEIC || normalized === MimeTypeImage.HEIF;
+}
diff --git a/tools/ui/src/lib/utils/process-uploaded-files.ts b/tools/ui/src/lib/utils/process-uploaded-files.ts
index d0110f6cbc..91b619cfb9 100644
--- a/tools/ui/src/lib/utils/process-uploaded-files.ts
+++ b/tools/ui/src/lib/utils/process-uploaded-files.ts
@@ -1,5 +1,6 @@
import { isSvgMimeType, svgBase64UrlToPngDataURL } from './svg-to-png';
import { isWebpMimeType, webpBase64UrlToPngDataURL } from './webp-to-png';
+import { heicFileToJpegDataURL, isHeicMimeType } from './heic-to-jpeg';
import { FileTypeCategory } from '$lib/enums';
import { SETTINGS_KEYS } from '$lib/constants';
import { modelsStore } from '$lib/stores/models.svelte';
@@ -68,7 +69,7 @@ export async function processFilesToChatUploaded(
if (getFileTypeCategory(file.type) === FileTypeCategory.IMAGE) {
let preview = await readFileAsDataURL(file);
- // Normalize SVG and WebP to PNG in previews
+ // Normalize SVG and WebP to PNG, and HEIC to compressed JPEG, in previews
if (isSvgMimeType(file.type)) {
try {
preview = await svgBase64UrlToPngDataURL(preview);
@@ -81,6 +82,13 @@ export async function processFilesToChatUploaded(
} catch (err) {
console.error('Failed to convert WebP to PNG:', err);
}
+ } else if (isHeicMimeType(file.type)) {
+ try {
+ preview = await heicFileToJpegDataURL(file);
+ } catch (err) {
+ console.error('Failed to convert HEIC to PNG:', err);
+ continue;
+ }
}
results.push({ ...base, preview });