High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat: warn when losing transparency during thumbnail generation (#26243)
Min Idzelis430638e129c1
diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.tsindex 33025e73cf..e3e78b3238 100644--- a/server/src/repositories/media.repository.ts+++ b/server/src/repositories/media.repository.ts@@ -309,9 +309,9 @@ export class MediaRepository {});}- async getImageDimensions(input: string | Buffer): Promise<ImageDimensions> {- const { width = 0, height = 0 } = await sharp(input).metadata();- return { width, height };+ async getImageMetadata(input: string | Buffer): Promise<ImageDimensions & { isTransparent: boolean }> {+ const { width = 0, height = 0, hasAlpha = false } = await sharp(input).metadata();+ return { width, height, isTransparent: hasAlpha };}private configureFfmpegCall(input: string, output: string | Writable, options: TranscodeCommand) {diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.tsindex 399eb5d6a0..368ece625c 100644--- a/server/src/services/media.service.spec.ts+++ b/server/src/services/media.service.spec.ts@@ -348,6 +348,7 @@ describe(MediaService.name, () => {: { data: fullsizeBuffer, info: rawInfo as OutputInfo }, // buffer implies embedded image extracted),);+ mocks.media.getImageMetadata.mockResolvedValue({ width: 100, height: 100, isTransparent: false });});it('should skip thumbnail generation if asset not found', async () => {@@ -857,7 +858,7 @@ describe(MediaService.name, () => {.exif({ fileSizeInByte: 5000, profileDescription: 'Adobe RGB', bitsPerSample: 14, orientation: undefined }).build();mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });- mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });+ mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false });mocks.systemMetadata.get.mockResolvedValue({ image: { extractEmbedded: true } });mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset);@@ -871,12 +872,39 @@ describe(MediaService.name, () => {});});+ it('should not check transparency metadata for raw files without extracted images', async () => {+ const asset = AssetFactory.from({ originalFileName: 'file.dng' })+ .exif({ fileSizeInByte: 5000, profileDescription: 'Adobe RGB', bitsPerSample: 14, orientation: undefined })+ .build();+ mocks.systemMetadata.get.mockResolvedValue({ image: { extractEmbedded: false } });+ mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset);++ await sut.handleGenerateThumbnails({ id: asset.id });++ expect(mocks.media.getImageMetadata).not.toHaveBeenCalled();+ });++ it('should not check transparency metadata for raw files with extracted images', async () => {+ const asset = AssetFactory.from({ originalFileName: 'file.dng' })+ .exif({ fileSizeInByte: 5000, profileDescription: 'Adobe RGB', bitsPerSample: 14, orientation: undefined })+ .build();+ mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });+ mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false });+ mocks.systemMetadata.get.mockResolvedValue({ image: { extractEmbedded: true } });+ mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset);++ await sut.handleGenerateThumbnails({ id: asset.id });++ expect(mocks.media.getImageMetadata).toHaveBeenCalledOnce();+ expect(mocks.media.getImageMetadata).toHaveBeenCalledWith(extractedBuffer);+ });+it('should resize original image if embedded image is too small', async () => {const asset = AssetFactory.from({ originalFileName: 'file.dng' }).exif({ fileSizeInByte: 5000, profileDescription: 'Adobe RGB', bitsPerSample: 14, orientation: undefined }).build();mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });- mocks.media.getImageDimensions.mockResolvedValue({ width: 1000, height: 1000 });+ mocks.media.getImageMetadata.mockResolvedValue({ width: 1000, height: 1000, isTransparent: false });mocks.systemMetadata.get.mockResolvedValue({ image: { extractEmbedded: true } });mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset);@@ -970,7 +998,7 @@ describe(MediaService.name, () => {image: { fullsize: { enabled: true, format: ImageFormat.Webp }, extractEmbedded: true },});mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });- mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });+ mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false });mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset);await sut.handleGenerateThumbnails({ id: asset.id });@@ -1008,7 +1036,7 @@ describe(MediaService.name, () => {image: { fullsize: { enabled: true, format: ImageFormat.Webp }, extractEmbedded: true },});mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jxl });- mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });+ mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false });mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset);await sut.handleGenerateThumbnails({ id: asset.id });@@ -1056,7 +1084,7 @@ describe(MediaService.name, () => {mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true }, extractEmbedded: false } });mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });- mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });+ mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false });mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset);await sut.handleGenerateThumbnails({ id: asset.id });@@ -1100,7 +1128,7 @@ describe(MediaService.name, () => {it('should generate full-size preview from non-web-friendly images', async () => {mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } });mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });- mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });+ mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false });// HEIF/HIF image taken by cameras are not web-friendly, only has limited support on Safari.const asset = AssetFactory.from({ originalFileName: 'image.hif' }).exif({@@ -1139,7 +1167,7 @@ describe(MediaService.name, () => {const asset = AssetFactory.from().exif().build();mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } });mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });- mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });+ mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false });mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(asset);await sut.handleGenerateThumbnails({ id: asset.id });@@ -1162,7 +1190,7 @@ describe(MediaService.name, () => {it('should always generate full-size preview from non-web-friendly panoramas', async () => {mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } });mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });- mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });+ mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false });mocks.media.copyTagGroup.mockResolvedValue(true);const asset = AssetFactory.from({ originalFileName: 'panorama.tif' })@@ -1208,7 +1236,7 @@ describe(MediaService.name, () => {image: { fullsize: { enabled: true, format: ImageFormat.Webp, quality: 90 } },});mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });- mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });+ mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false });// HEIF/HIF image taken by cameras are not web-friendly, only has limited support on Safari.const asset = AssetFactory.from({ originalFileName: 'image.hif' }).exif({@@ -1248,7 +1276,7 @@ describe(MediaService.name, () => {image: { fullsize: { enabled: true, format: ImageFormat.Jpeg, progressive: true } },});mocks.media.extract.mockResolvedValue({ buffer: extractedBuffer, format: RawExtractedFormat.Jpeg });- mocks.media.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 });+ mocks.media.getImageMetadata.mockResolvedValue({ width: 3840, height: 2160, isTransparent: false });const asset = AssetFactory.from({ originalFileName: 'image.hif' }).exif({fileSizeInByte: 5000,@@ -1286,6 +1314,7 @@ describe(MediaService.name, () => {: { data: fullsizeBuffer, info: rawInfo as OutputInfo }, // buffer implies embedded image extracted),);+ mocks.media.getImageMetadata.mockResolvedValue({ width: 100, height: 100, isTransparent: false });});it('should skip videos', async () => {@@ -1719,7 +1748,7 @@ describe(MediaService.name, () => {const info = { width: 2160, height: 3840 } as OutputInfo;mocks.media.extract.mockResolvedValue({ buffer: extracted, format: RawExtractedFormat.Jpeg });mocks.media.decodeImage.mockResolvedValue({ data, info });- mocks.media.getImageDimensions.mockResolvedValue(info);+ mocks.media.getImageMetadata.mockResolvedValue({ width: 2160, height: 3840, isTransparent: false });await expect(sut.handleGeneratePersonThumbnail({ id: personStub.primaryPerson.id })).resolves.toBe(JobStatus.Success,@@ -1802,7 +1831,7 @@ describe(MediaService.name, () => {const info = { width: 1000, height: 1000 } as OutputInfo;mocks.media.decodeImage.mockResolvedValue({ data, info });mocks.media.extract.mockResolvedValue({ buffer: extracted, format: RawExtractedFormat.Jpeg });- mocks.media.getImageDimensions.mockResolvedValue(info);+ mocks.media.getImageMetadata.mockResolvedValue({ width: 1000, height: 1000, isTransparent: false });await expect(sut.handleGeneratePersonThumbnail({ id: personStub.primaryPerson.id })).resolves.toBe(JobStatus.Success,diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.tsindex 5fa72cf117..153083142d 100644--- a/server/src/services/media.service.ts+++ b/server/src/services/media.service.ts@@ -280,14 +280,20 @@ export class MediaService extends BaseService {useEdits;const convertFullsize = generateFullsize && (!extracted || !mimeTypes.isWebSupportedImage(` .${extracted.format}`));+ const thumbSource = extracted ? extracted.buffer : asset.originalPath;const { data, info, colorspace } = await this.decodeImage(- extracted ? extracted.buffer : asset.originalPath,+ thumbSource,// only specify orientation to extracted images which don't have EXIF orientation data// or it can double rotate the imageextracted ? asset.exifInfo : { ...asset.exifInfo, orientation: null },convertFullsize ? undefined : image.preview.size,);+ let isTransparent = false;+ if (!extracted && mimeTypes.canBeTransparent(asset.originalPath)) {+ ({ isTransparent } = await this.mediaRepository.getImageMetadata(asset.originalPath));+ }+return {extracted,data,@@ -295,50 +301,61 @@ export class MediaService extends BaseService {colorspace,convertFullsize,generateFullsize,+ isTransparent,};}private async generateImageThumbnails(asset: ThumbnailAsset, { image }: SystemConfig, useEdits: boolean = false) {+ // Handle embedded preview extraction for RAW files+ const extractedImage = await this.extractOriginalImage(asset, image, useEdits);+ const { info, data, colorspace, generateFullsize, convertFullsize, extracted, isTransparent } = extractedImage;++ const previewFormat = image.preview.format;+ this.warnOnTransparencyLoss(isTransparent, previewFormat, asset.id);++ const thumbnailFormat = image.thumbnail.format;+ this.warnOnTransparencyLoss(isTransparent, thumbnailFormat, asset.id);+const previewFile = this.getImageFile(asset, {fileType: AssetFileType.Preview,- format: image.preview.format,+ format: previewFormat,isEdited: useEdits,- isProgressive: !!image.preview.progressive && image.preview.format !== ImageFormat.Webp,+ isProgressive: !!image.preview.progressive && previewFormat !== ImageFormat.Webp,});const thumbnailFile = this.getImageFile(asset, {fileType: AssetFileType.Thumbnail,- format: image.thumbnail.format,+ format: thumbnailFormat,isEdited: useEdits,- isProgressive: !!image.thumbnail.progressive && image.thumbnail.format !== ImageFormat.Webp,+ isProgressive: !!image.thumbnail.progressive && thumbnailFormat !== ImageFormat.Webp,});this.storageCore.ensureFolders(previewFile.path);- // Handle embedded preview extraction for RAW files- const extractedImage = await this.extractOriginalImage(asset, image, useEdits);- const { info, data, colorspace, generateFullsize, convertFullsize, extracted } = extractedImage;-// generate final images- const thumbnailOptions = { colorspace, processInvalidImages: false, raw: info, edits: useEdits ? asset.edits : [] };+ const baseOptions = { colorspace, processInvalidImages: false, raw: info, edits: useEdits ? asset.edits : [] };+ const thumbnailOptions = { ...image.thumbnail, ...baseOptions, format: thumbnailFormat };+ const previewOptions = { ...image.preview, ...baseOptions, format: previewFormat };const promises = [- this.mediaRepository.generateThumbhash(data, thumbnailOptions),- this.mediaRepository.generateThumbnail(data, { ...image.thumbnail, ...thumbnailOptions }, thumbnailFile.path),- this.mediaRepository.generateThumbnail(data, { ...image.preview, ...thumbnailOptions }, previewFile.path),+ this.mediaRepository.generateThumbhash(data, baseOptions),+ this.mediaRepository.generateThumbnail(data, thumbnailOptions, thumbnailFile.path),+ this.mediaRepository.generateThumbnail(data, previewOptions, previewFile.path),];let fullsizeFile: UpsertFileOptions | undefined;if (convertFullsize) {+ const fullsizeFormat = image.fullsize.format;+ this.warnOnTransparencyLoss(isTransparent, fullsizeFormat, asset.id);// convert a new fullsize image from the same source as the thumbnailfullsizeFile = this.getImageFile(asset, {fileType: AssetFileType.FullSize,- format: image.fullsize.format,+ format: fullsizeFormat,isEdited: useEdits,- isProgressive: !!image.fullsize.progressive && image.fullsize.format !== ImageFormat.Webp,+ isProgressive: !!image.fullsize.progressive && fullsizeFormat !== ImageFormat.Webp,});const fullsizeOptions = {- format: image.fullsize.format,+ ...baseOptions,+ format: fullsizeFormat,quality: image.fullsize.quality,progressive: image.fullsize.progressive,- ...thumbnailOptions,};promises.push(this.mediaRepository.generateThumbnail(data, fullsizeOptions, fullsizeFile.path));} else if (generateFullsize && extracted && extracted.format === RawExtractedFormat.Jpeg) {@@ -758,7 +775,7 @@ export class MediaService extends BaseService {}private async shouldUseExtractedImage(extractedPathOrBuffer: string | Buffer, targetSize: number) {- const { width, height } = await this.mediaRepository.getImageDimensions(extractedPathOrBuffer);+ const { width, height } = await this.mediaRepository.getImageMetadata(extractedPathOrBuffer);const extractedSize = Math.min(width, height);return extractedSize >= targetSize;}@@ -857,6 +874,14 @@ export class MediaService extends BaseService {return generated;}+ private warnOnTransparencyLoss(isTransparent: boolean, format: ImageFormat, assetId: string) {+ if (isTransparent && format === ImageFormat.Jpeg) {+ this.logger.warn(+ `Asset ${assetId} has transparency but the configured format is ${format} which does not support it, consider using a format that does, such as ${ImageFormat.Webp}`,+ );+ }+ }+private getImageFile(asset: ThumbnailPathEntity, options: ImagePathOptions & { isProgressive: boolean }) {const path = StorageCore.getImagePath(asset, options);return {diff --git a/server/src/utils/mime-types.spec.ts b/server/src/utils/mime-types.spec.tsindex b0e31afe39..862ed310bc 100644--- a/server/src/utils/mime-types.spec.ts+++ b/server/src/utils/mime-types.spec.ts@@ -153,6 +153,33 @@ describe('mimeTypes', () => {}});+ describe('canBeTransparent', () => {+ for (const img of [+ 'a.avif',+ 'a.bmp',+ 'a.gif',+ 'a.heic',+ 'a.heif',+ 'a.hif',+ 'a.jxl',+ 'a.png',+ 'a.svg',+ 'a.tif',+ 'a.tiff',+ 'a.webp',+ ]) {+ it(`should return true for ${img}`, () => {+ expect(mimeTypes.canBeTransparent(img)).toBe(true);+ });+ }++ for (const img of ['a.jpg', 'a.jpeg', 'a.jpe', 'a.insp', 'a.jp2', 'a.cr3', 'a.dng', 'a.nef', 'a.arw']) {+ it(`should return false for ${img}`, () => {+ expect(mimeTypes.canBeTransparent(img)).toBe(false);+ });+ }+ });+describe('animated image', () => {for (const img of ['a.avif', 'a.gif', 'a.webp']) {it('should identify animated image mime types as such', () => {diff --git a/server/src/utils/mime-types.ts b/server/src/utils/mime-types.tsindex 4e91bbd7f1..43421e7937 100644--- a/server/src/utils/mime-types.ts+++ b/server/src/utils/mime-types.ts@@ -77,6 +77,21 @@ const extensionOverrides: Record<string, string> = {'image/jpeg': '.jpg',};+const transparentCapableExtensions = new Set([+ '.avif',+ '.bmp',+ '.gif',+ '.heic',+ '.heif',+ '.hif',+ '.jxl',+ '.png',+ '.svg',+ '.tif',+ '.tiff',+ '.webp',+]);+const profileExtensions = new Set(['.avif', '.dng', '.heic', '.heif', '.jpeg', '.jpg', '.png', '.webp', '.svg']);const profile: Record<string, string[]> = Object.fromEntries(Object.entries(image).filter(([key]) => profileExtensions.has(key)),@@ -134,6 +149,7 @@ export const mimeTypes = {isProfile: (filename: string) => isType(filename, profile),isSidecar: (filename: string) => isType(filename, sidecar),isVideo: (filename: string) => isType(filename, video),+ canBeTransparent: (filename: string) => transparentCapableExtensions.has(extname(filename).toLowerCase()),isRaw: (filename: string) => isType(filename, raw),lookup,/** return an extension (including a leading `.`) for a mime-type */diff --git a/server/test/repositories/media.repository.mock.ts b/server/test/repositories/media.repository.mock.tsindex b6b1e82b52..bd8deb4b3a 100644--- a/server/test/repositories/media.repository.mock.ts+++ b/server/test/repositories/media.repository.mock.ts@@ -12,6 +12,6 @@ export const newMediaRepositoryMock = (): Mocked<RepositoryInterface<MediaReposiextract: vitest.fn().mockResolvedValue(null),probe: vitest.fn(),transcode: vitest.fn(),- getImageDimensions: vitest.fn(),+ getImageMetadata: vitest.fn(),};};