High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
chore(server): better xmp testing (#31928)
Jonathan Jogenforsa010802431be
diff --git a/e2e/src/specs/server/api/asset.e2e-spec.ts b/e2e/src/specs/server/api/asset.e2e-spec.tsindex 6eb0dc4124..c29d105fb7 100644--- a/e2e/src/specs/server/api/asset.e2e-spec.ts+++ b/e2e/src/specs/server/api/asset.e2e-spec.ts@@ -380,44 +380,6 @@ describe('/asset', () => {expect(body).toMatchObject({ id: user1Assets[0].id, livePhotoVideoId: null });});- it.skip('should update date time original when sidecar file contains DateTimeOriginal', async () => {- const sidecarData = `<?xpacket begin='?' id='W5M0MpCehiHzreSzNTczkc9d'?>-<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='Image::ExifTool 12.40'>-<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#'>- <rdf:Description rdf:about=''- xmlns:exif='http://ns.adobe.com/exif/1.0/'>- <exif:ExifVersion>0220</exif:ExifVersion> <exif:DateTimeOriginal>2024-07-11T10:32:52Z</exif:DateTimeOriginal>- <exif:GPSVersionID>2.3.0.0</exif:GPSVersionID>- </rdf:Description>-</rdf:RDF>-</x:xmpmeta>-<?xpacket end='w'?>`;-- const { id } = await utils.createAsset(user1.accessToken, {- sidecarData: {- bytes: Buffer.from(sidecarData),- filename: 'example.xmp',- },- });- await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');-- const assetInfo = await utils.getAssetInfo(user1.accessToken, id);- expect(assetInfo.exifInfo?.dateTimeOriginal).toBe('2024-07-11T10:32:52+00:00');-- const { status, body } = await request(app)- .put(`/assets/${id}`)- .set('Authorization', `Bearer ${user1.accessToken}`)- .send({ dateTimeOriginal: '2023-11-19T18:11:00.000-07:00' });-- expect(body).toMatchObject({- id,- exifInfo: expect.objectContaining({- dateTimeOriginal: '2023-11-20T01:11:00+00:00',- }),- });- expect(status).toEqual(200);- });-it('should update gps data', async () => {const { status, body } = await request(app).put(`/assets/${user1Assets[0].id}`)diff --git a/server/src/repositories/storage.repository.spec.ts b/server/src/repositories/storage.repository.spec.tsindex 3d1e503ed4..a6486532ce 100644--- a/server/src/repositories/storage.repository.spec.ts+++ b/server/src/repositories/storage.repository.spec.ts@@ -1,4 +1,8 @@import mockfs from 'mock-fs';+import { R_OK } from 'node:constants';+import { mkdtempDisposable, writeFile } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';import { vitest } from 'vitest';import { CrawlOptionsDto } from 'src/dtos/library.dto.js';import { LoggingRepository } from 'src/repositories/logging.repository.js';@@ -231,6 +235,52 @@ describe(StorageRepository.name, () => {}});+ describe('walk', () => {+ it.each([+ { exclusionPatterns: [] },+ { exclusionPatterns: ['**/*.xmp'] },+ { exclusionPatterns: ['**/excluded/**'] },+ ])('should only return assets and respect exclusions: $exclusionPatterns', async ({ exclusionPatterns }) => {+ mockfs({+ '/photos/photo.jpg': '',+ '/photos/photo.nef': '',+ '/photos/photo.jpg.xmp': '',+ '/photos/photo.xmp': '',+ '/photos/excluded/photo.jpg': '',+ '/photos/excluded/photo.xmp': '',+ });++ const batches = await Array.fromAsync(sut.walk({ pathsToCrawl: ['/photos'], exclusionPatterns, take: 1 }));++ expect(batches.every((batch) => batch.length === 1)).toBe(true);+ expect(batches.flat().toSorted()).toEqual(+ [+ '/photos/photo.jpg',+ '/photos/photo.nef',+ ...(exclusionPatterns.includes('**/excluded/**') ? [] : ['/photos/excluded/photo.jpg']),+ ].toSorted(),+ );+ });+ });++ describe('checkFileExists', () => {+ it.for(['PHOTO.xmp', 'photo.XMP'])(+ 'should not match %s with different case on a case-sensitive filesystem',+ async (filename, { skip }) => {+ await using tempDir = await mkdtempDisposable(join(tmpdir(), 'immich-storage-sidecar-'));+ await writeFile(join(tempDir.path, 'case-probe'), 'test');+ if (await sut.checkFileExists(join(tempDir.path, 'CASE-PROBE'), R_OK)) {+ skip();+ }+ const candidate = join(tempDir.path, filename);+ await writeFile(candidate, 'test');++ await expect(sut.checkFileExists(candidate, R_OK)).resolves.toBe(true);+ await expect(sut.checkFileExists(join(tempDir.path, 'photo.xmp'), R_OK)).resolves.toBe(false);+ },+ );+ });+describe('watch', () => {it('should register event handlers and close the watcher', async () => {const onReady = vitest.fn();diff --git a/server/src/services/job.service.spec.ts b/server/src/services/job.service.spec.tsindex baaf97cac7..b2dcc15b96 100644--- a/server/src/services/job.service.spec.ts+++ b/server/src/services/job.service.spec.ts@@ -32,15 +32,32 @@ describe(JobService.name, () => {expect(mocks.logger.error).not.toHaveBeenCalled();});- const tests: Array<{ item: JobItem; jobs: JobName[]; stub?: any }> = [- {- item: { name: JobName.SidecarCheck, data: { id: 'asset-1' } },- jobs: [JobName.AssetExtractMetadata],- },- {- item: { name: JobName.SidecarCheck, data: { id: 'asset-1' } },- jobs: [JobName.AssetExtractMetadata],+ it.each([JobStatus.Success, JobStatus.Skipped])(+ 'should queue metadata extraction after a %s sidecar check and preserve its source',+ async (status) => {+ mocks.job.run.mockResolvedValue(status);+ const job: JobItem = { name: JobName.SidecarCheck, data: { id: 'asset-1', source: 'upload' } };++ await sut.onJobRun(QueueName.Sidecar, job);++ expect(mocks.job.queue).toHaveBeenCalledExactlyOnceWith({+ name: JobName.AssetExtractMetadata,+ data: { id: 'asset-1', source: 'upload' },+ });+ expect(mocks.job.queueAll).not.toHaveBeenCalled();},+ );++ it('should not queue metadata extraction after a failed sidecar check', async () => {+ mocks.job.run.mockResolvedValue(JobStatus.Failed);++ await sut.onJobRun(QueueName.Sidecar, { name: JobName.SidecarCheck, data: { id: 'asset-1', source: 'upload' } });++ expect(mocks.job.queue).not.toHaveBeenCalled();+ expect(mocks.job.queueAll).not.toHaveBeenCalled();+ });++ const tests: Array<{ item: JobItem; jobs: JobName[]; stub?: any }> = [{item: { name: JobName.StorageTemplateMigrationSingle, data: { id: 'asset-1', source: 'upload' } },jobs: [JobName.AssetGenerateThumbnails],diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.tsindex 53b3e7035c..2c281c7307 100644--- a/server/src/services/library.service.spec.ts+++ b/server/src/services/library.service.spec.ts@@ -487,6 +487,7 @@ describe(LibraryService.name, () => {await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.Success);expect(mocks.asset.updateAll).not.toHaveBeenCalled();+ expect(mocks.job.queueAll).not.toHaveBeenCalled();});it('should not touch fileCreatedAt when un-trashing an asset previously marked as offline', async () => {diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.tsindex f8a2807427..31e2a427a8 100644--- a/server/src/services/metadata.service.spec.ts+++ b/server/src/services/metadata.service.spec.ts@@ -1,5 +1,6 @@import { BinaryField, ExifDateTime } from 'exiftool-vendored';import { DateTime } from 'luxon';+import { R_OK } from 'node:constants';import { randomBytes } from 'node:crypto';import { Stats } from 'node:fs';import { defaults } from 'src/dtos/config.dto.js';@@ -187,12 +188,12 @@ describe(MetadataService.name, () => {expect(mocks.asset.update).not.toHaveBeenCalled();});- it('should handle a date in a sidecar file', async () => {+ it.each(['CreationDate', 'DateTimeOriginal'] as const)('should prefer %s from a sidecar file', async (tag) => {const originalDate = new Date('2023-11-21T16:13:17.517Z');const sidecarDate = new Date('2022-01-01T00:00:00.000Z');const asset = AssetFactory.from().file({ type: AssetFileType.Sidecar }).build();mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));- mockReadTags({ CreationDate: originalDate.toISOString() }, { CreationDate: sidecarDate.toISOString() });+ mockReadTags({ [tag]: originalDate.toISOString() }, { [tag]: sidecarDate.toISOString() });await sut.handleMetadataExtraction({ id: asset.id });expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);@@ -1920,6 +1921,7 @@ describe(MetadataService.name, () => {type: AssetFileType.Sidecar,path: '/path/to/IMG_123.jpg.xmp',});+ expect(mocks.storage.checkFileExists).toHaveBeenCalledExactlyOnceWith('/path/to/IMG_123.jpg.xmp', R_OK);});it('should detect a new sidecar at .xmp', async () => {@@ -1939,34 +1941,163 @@ describe(MetadataService.name, () => {type: AssetFileType.Sidecar,path: '/path/to/IMG_123.xmp',});+ expect(mocks.storage.checkFileExists.mock.calls).toEqual([+ ['/path/to/IMG_123.jpg.xmp', R_OK],+ ['/path/to/IMG_123.xmp', R_OK],+ ]);});- it('should unset sidecar path if file no longer exist', async () => {+ it('should prefer the extended filename when both candidates exist', async () => {+ const asset = forSidecarJob();+ const candidates = new Set(['/path/to/IMG_123.jpg.xmp', '/path/to/IMG_123.xmp']);+ mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);+ mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(candidates.has(path)));++ await sut.handleSidecarCheck({ id: asset.id });++ expect(mocks.asset.upsertFile).toHaveBeenCalledExactlyOnceWith({+ assetId: asset.id,+ type: AssetFileType.Sidecar,+ path: '/path/to/IMG_123.jpg.xmp',+ });+ expect(mocks.storage.checkFileExists).toHaveBeenCalledExactlyOnceWith('/path/to/IMG_123.jpg.xmp', R_OK);+ });++ it('should preserve an existing association ahead of both conventional candidates', async () => {+ const existing = '/elsewhere/custom.xmp';const asset = forSidecarJob({- originalPath: '/path/to/IMG_123.jpg',- files: [{ id: 'sidecar', path: '/path/to/IMG_123.jpg.xmp', type: AssetFileType.Sidecar, isEdited: false }],+ files: [{ id: 'sidecar', type: AssetFileType.Sidecar, path: existing, isEdited: false }],});+ const candidates = new Set([existing, '/path/to/IMG_123.jpg.xmp', '/path/to/IMG_123.xmp']);+ mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);+ mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(candidates.has(path)));++ await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Skipped);++ expect(mocks.storage.checkFileExists).toHaveBeenCalledExactlyOnceWith(existing, R_OK);+ expect(mocks.asset.upsertFile).not.toHaveBeenCalled();+ expect(mocks.asset.deleteFile).not.toHaveBeenCalled();+ });++ it.each(['/path/to/IMG_123.jpg.xmp', '/path/to/IMG_123.xmp'])(+ 'should replace a missing association with %s',+ async (replacement) => {+ const existing = '/elsewhere/missing.xmp';+ const asset = forSidecarJob({+ files: [{ id: 'sidecar', type: AssetFileType.Sidecar, path: existing, isEdited: false }],+ });+ mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);+ mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === replacement));++ await sut.handleSidecarCheck({ id: asset.id });++ expect(mocks.storage.checkFileExists).toHaveBeenNthCalledWith(1, existing, R_OK);+ expect(mocks.asset.upsertFile).toHaveBeenCalledExactlyOnceWith({+ assetId: asset.id,+ type: AssetFileType.Sidecar,+ path: replacement,+ });+ expect(mocks.asset.deleteFile).not.toHaveBeenCalled();+ },+ );++ it('should not create an association when no sidecar exists', async () => {+ const asset = forSidecarJob();mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);mocks.storage.checkFileExists.mockResolvedValue(false);- await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Success);+ await sut.handleSidecarCheck({ id: asset.id });- expect(mocks.asset.deleteFile).toHaveBeenCalledWith({ assetId: asset.id, type: AssetFileType.Sidecar });+ expect(mocks.storage.checkFileExists.mock.calls).toEqual([+ ['/path/to/IMG_123.jpg.xmp', R_OK],+ ['/path/to/IMG_123.xmp', R_OK],+ ]);+ expect(mocks.asset.upsertFile).not.toHaveBeenCalled();});- it('should do nothing if the sidecar file still exists', async () => {+ it.each(['/photos/Photo.JPG.xmp', '/photos/Photo.xmp'])(+ 'should preserve filename case when discovering %s',+ async (sidecarPath) => {+ const asset = forSidecarJob({ originalPath: '/photos/Photo.JPG' });+ mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);+ mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === sidecarPath));++ await sut.handleSidecarCheck({ id: asset.id });++ expect(mocks.asset.upsertFile).toHaveBeenCalledExactlyOnceWith({+ assetId: asset.id,+ type: AssetFileType.Sidecar,+ path: sidecarPath,+ });+ },+ );++ it.each([+ { filename: 'photo.edit.jpg', sidecar: 'photo.edit.xmp' },+ { filename: '.photo.jpg', sidecar: '.photo.xmp' },+ { filename: 'quoted"雪.jpg', sidecar: 'quoted"雪.xmp' },+ ])('should preserve the basename when finding the fallback for $filename', async ({ filename, sidecar }) => {+ const asset = forSidecarJob({ originalPath: `/photos/${filename}` });+ const sidecarPath = `/photos/${sidecar}`;+ mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);+ mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === sidecarPath));++ await sut.handleSidecarCheck({ id: asset.id });++ expect(mocks.asset.upsertFile).toHaveBeenCalledExactlyOnceWith({+ assetId: asset.id,+ type: AssetFileType.Sidecar,+ path: sidecarPath,+ });+ });++ it.each([false, true])(+ 'should resolve shared JPG and RAW sidecars independently (extended JPG sidecar: %s)',+ async (extended) => {+ const jpg = forSidecarJob({ originalPath: '/photos/photo.jpg' });+ const raw = forSidecarJob({ originalPath: '/photos/photo.nef' });+ const shared = '/photos/photo.xmp';+ const candidates = new Set([shared, ...(extended ? ['/photos/photo.jpg.xmp'] : [])]);+ mocks.assetJob.getForSidecarCheckJob.mockResolvedValueOnce(jpg).mockResolvedValueOnce(raw);+ mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(candidates.has(path)));++ await sut.handleSidecarCheck({ id: jpg.id });+ await sut.handleSidecarCheck({ id: raw.id });++ expect(mocks.asset.upsertFile.mock.calls).toEqual([+ [{ assetId: jpg.id, type: AssetFileType.Sidecar, path: extended ? '/photos/photo.jpg.xmp' : shared }],+ [{ assetId: raw.id, type: AssetFileType.Sidecar, path: shared }],+ ]);+ },+ );++ it('should not match xmp files across directories', async () => {+ const first = forSidecarJob({ originalPath: '/photos/first/photo.jpg' });+ const second = forSidecarJob({ originalPath: '/photos/second/photo.jpg' });+ mocks.assetJob.getForSidecarCheckJob.mockResolvedValueOnce(first).mockResolvedValueOnce(second);+ mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === '/photos/second/photo.xmp'));++ await sut.handleSidecarCheck({ id: first.id });+ await sut.handleSidecarCheck({ id: second.id });++ expect(mocks.asset.upsertFile).toHaveBeenCalledExactlyOnceWith({+ assetId: second.id,+ type: AssetFileType.Sidecar,+ path: '/photos/second/photo.xmp',+ });+ });++ it('should unset sidecar path if file no longer exist', async () => {const asset = forSidecarJob({originalPath: '/path/to/IMG_123.jpg',files: [{ id: 'sidecar', path: '/path/to/IMG_123.jpg.xmp', type: AssetFileType.Sidecar, isEdited: false }],});-mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);- mocks.storage.checkFileExists.mockResolvedValueOnce(true);+ mocks.storage.checkFileExists.mockResolvedValue(false);- await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Skipped);+ await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Success);- expect(mocks.asset.upsertFile).not.toHaveBeenCalled();- expect(mocks.asset.deleteFile).not.toHaveBeenCalled();+ expect(mocks.asset.deleteFile).toHaveBeenCalledWith({ assetId: asset.id, type: AssetFileType.Sidecar });});});diff --git a/server/test/medium/specs/repositories/asset.repository.spec.ts b/server/test/medium/specs/repositories/asset.repository.spec.tsindex 7a434d78df..11596da98b 100644--- a/server/test/medium/specs/repositories/asset.repository.spec.ts+++ b/server/test/medium/specs/repositories/asset.repository.spec.ts@@ -1,5 +1,5 @@import { Kysely } from 'kysely';-import { AssetOrder, AssetOrderBy, AssetVisibility } from 'src/enum.js';+import { AssetFileType, AssetOrder, AssetOrderBy, AssetVisibility } from 'src/enum.js';import { AssetRepository } from 'src/repositories/asset.repository.js';import { LoggingRepository } from 'src/repositories/logging.repository.js';import { DB } from 'src/schema/index.js';@@ -64,6 +64,31 @@ const keyframeRow = (assetId: string, n: number) => ({});describe(AssetRepository.name, () => {+ describe('deleteFile', () => {+ it('should delete only the requested asset and file type', async () => {+ const { ctx, sut } = setup();+ const { user } = await ctx.newUser();+ const { asset } = await ctx.newAsset({ ownerId: user.id });+ const { asset: other } = await ctx.newAsset({ ownerId: user.id });+ await sut.upsertFiles([+ { assetId: asset.id, type: AssetFileType.Sidecar, path: '/photos/photo.jpg.xmp' },+ { assetId: asset.id, type: AssetFileType.Thumbnail, path: '/photos/thumbnail.jpg' },+ { assetId: other.id, type: AssetFileType.Sidecar, path: '/photos/other.jpg.xmp' },+ ]);++ await sut.deleteFile({ assetId: asset.id, type: AssetFileType.Sidecar });++ await expect(+ ctx.database+ .selectFrom('asset_file')+ .where('assetId', 'in', [asset.id, other.id])+ .select('path')+ .orderBy('path')+ .execute(),+ ).resolves.toEqual([{ path: '/photos/other.jpg.xmp' }, { path: '/photos/thumbnail.jpg' }]);+ });+ });+describe('getTimeBucket', () => {it('should order assets by local day first and fileCreatedAt within each day', async () => {const { ctx, sut } = setup();diff --git a/server/test/medium/specs/services/library.service.spec.ts b/server/test/medium/specs/services/library.service.spec.tsindex 42730db2d3..c250287984 100644--- a/server/test/medium/specs/services/library.service.spec.ts+++ b/server/test/medium/specs/services/library.service.spec.ts@@ -1,11 +1,11 @@import { Kysely } from 'kysely';import { existsSync } from 'node:fs';-import { copyFile, mkdir, mkdtemp, rm, utimes, writeFile } from 'node:fs/promises';+import { mkdir, mkdtemp, rm, utimes, writeFile } from 'node:fs/promises';import { tmpdir } from 'node:os';import { dirname, join, resolve } from 'node:path';import type { SystemConfig } from 'src/dtos/config.dto.js';import { StorageCore } from 'src/cores/storage.core.js';-import { AssetStatus, JobName, JobStatus } from 'src/enum.js';+import { AssetFileType, AssetStatus, JobName, JobStatus } from 'src/enum.js';import { AssetJobRepository } from 'src/repositories/asset-job.repository.js';import { AssetRepository } from 'src/repositories/asset.repository.js';import { CronRepository } from 'src/repositories/cron.repository.js';@@ -18,8 +18,9 @@ import { LoggingRepository } from 'src/repositories/logging.repository.js';import { StorageRepository } from 'src/repositories/storage.repository.js';import { DB } from 'src/schema/index.js';import { LibraryService } from 'src/services/library.service.js';+import { MetadataService } from 'src/services/metadata.service.js';import { systemConfigStub } from 'test/fixtures/system-config.stub.js';-import { MediumTestContext, testAssetsDir } from 'test/medium.factory.js';+import { MediumTestContext, newMediumService } from 'test/medium.factory.js';import { newUuid } from 'test/small.factory.js';import { getKyselyDB } from 'test/utils.js';@@ -37,14 +38,9 @@ const createFile = async (filePath: string, modifiedAt: Date = fileModifiedAt) =return filePath;};-const copyTestAsset = async (source: string, filePath: string, modifiedAt: Date = fileModifiedAt) => {- await mkdir(dirname(filePath), { recursive: true });- await copyFile(join(testAssetsDir, source), filePath);- await utimes(filePath, modifiedAt, modifiedAt);- return filePath;-};-class LibraryTestContext extends MediumTestContext<typeof LibraryService> {+ private metadataService: MetadataService;+constructor(database: Kysely<DB>) {super(LibraryService, {database,@@ -57,6 +53,12 @@ class LibraryTestContext extends MediumTestContext<typeof LibraryService> {jobs.queueAll.mockResolvedValue();this.getMock(EventRepository).emit.mockResolvedValue();++ this.metadataService = newMediumService(MetadataService, {+ database,+ real: [AssetRepository, AssetJobRepository, StorageRepository],+ mock: [LoggingRepository],+ }).sut;}async createLibrary(options: { importPaths?: string[]; exclusionPatterns?: string[] } = {}) {@@ -74,20 +76,45 @@ class LibraryTestContext extends MediumTestContext<typeof LibraryService> {const jobs = this.getMock(JobRepository);jobs.queue.mockClear();+ jobs.queueAll.mockClear();await this.sut.handleQueueSyncFiles({ id: libraryId });for (const [job] of jobs.queue.mock.calls) {if (job.name === JobName.LibrarySyncFiles) {await this.sut.handleSyncFiles(job.data);}}+ await this.discoverSidecars();jobs.queue.mockClear();+ jobs.queueAll.mockClear();await this.sut.handleQueueSyncAssets({ id: libraryId });for (const [job] of jobs.queue.mock.calls) {if (job.name === JobName.LibrarySyncAssets) {await this.sut.handleSyncAssets(job.data);}}+ await this.discoverSidecars();+ }++ private async discoverSidecars() {+ for (const [jobs] of this.getMock(JobRepository).queueAll.mock.calls) {+ for (const job of jobs) {+ if (job.name === JobName.SidecarCheck) {+ await this.metadataService.handleSidecarCheck(job.data);+ }+ }+ }+ }++ getSidecarPaths(libraryId: string) {+ return this.database+ .selectFrom('asset')+ .innerJoin('asset_file', 'asset_file.assetId', 'asset.id')+ .where('libraryId', '=', libraryId)+ .where('asset_file.type', '=', AssetFileType.Sidecar)+ .select(['originalPath', 'asset_file.path as sidecarPath'])+ .orderBy('originalPath')+ .execute();}/** The paths a library scan left visible, i.e. neither offline nor trashed */@@ -813,95 +840,38 @@ describe(LibraryService.name, () => {expect(updated).toEqual(expect.objectContaining({ isOffline: false }));expect(updated?.deletedAt).toBeInstanceOf(Date);});-- it('should queue sidecar checks for assets whose file changed', async () => {- const { sut, ctx } = setup();- const jobs = ctx.getMock(JobRepository);- const library = await ctx.createLibrary({ importPaths: [importPath] });- const rawPath = await copyTestAsset('formats/raw/Nikon/D80/glarus.nef', join(importPath, 'glarus.nef'));-- const { asset } = await ctx.newAsset({- ownerId: library.ownerId,- libraryId: library.id,- originalPath: rawPath,- // the file on disk has a newer modified time- fileModifiedAt: new Date(fileModifiedAt.valueOf() - 1000),- isExternal: true,- isOffline: false,- status: AssetStatus.Active,- });-- await expect(- sut.handleSyncAssets({- libraryId: library.id,- importPaths: library.importPaths,- exclusionPatterns: library.exclusionPatterns,- assetIds: [asset.id],- progressCounter: 1,- totalAssets: 1,- }),- ).resolves.toBe(JobStatus.Success);-- expect(jobs.queueAll).toHaveBeenCalledWith([- {- name: JobName.SidecarCheck,- data: { id: asset.id, source: 'upload' },- },- ]);- });-- it('should not queue sidecar checks for unchanged assets', async () => {- const { sut, ctx } = setup();- const jobs = ctx.getMock(JobRepository);- const library = await ctx.createLibrary({ importPaths: [importPath] });- const rawPath = await copyTestAsset('formats/raw/Nikon/D80/glarus.nef', join(importPath, 'glarus.nef'));-- const { asset } = await ctx.newAsset({- ownerId: library.ownerId,- libraryId: library.id,- originalPath: rawPath,- fileModifiedAt,- isExternal: true,- isOffline: false,- status: AssetStatus.Active,- });-- await expect(- sut.handleSyncAssets({- libraryId: library.id,- importPaths: library.importPaths,- exclusionPatterns: library.exclusionPatterns,- assetIds: [asset.id],- progressCounter: 1,- totalAssets: 1,- }),- ).resolves.toBe(JobStatus.Success);-- expect(jobs.queueAll).not.toHaveBeenCalled();- });});- describe('handleSyncFiles', () => {- it('should queue sidecar checks for newly imported assets', async () => {- const { sut, ctx } = setup();- const jobs = ctx.getMock(JobRepository);- const library = await ctx.createLibrary({ importPaths: [importPath] });- const rawPath = await copyTestAsset('formats/raw/Nikon/D80/glarus.nef', join(importPath, 'glarus.nef'));-- await expect(- sut.handleSyncFiles({- libraryId: library.id,- paths: [rawPath],- progressCounter: 1,- }),- ).resolves.toBe(JobStatus.Success);+ describe('sidecar discovery during library scans', () => {+ it('should only associate with sidecars in the same directory across multiple scan batches and import paths', async () => {+ const { ctx } = setup();+ const secondImportPath = join(importRoot, 'second');+ const library = await ctx.createLibrary({+ importPaths: [importPath, secondImportPath],+ });+ const expected = [];+ for (const directory of [importPath, secondImportPath]) {+ for (const name of ['a', 'b', 'c']) {+ const originalPath = await createFile(join(directory, name, 'photo.jpg'));+ const sidecarPath = await createFile(join(directory, name, 'photo.xmp'));+ expected.push({ originalPath, sidecarPath });+ }+ }+ const storage = ctx.get(StorageRepository);+ const walk = storage.walk.bind(storage);+ const walker = vi.spyOn(storage, 'walk').mockImplementation((options) => walk({ ...options, take: 2 }));+ const syncFiles = vi.spyOn(ctx.sut, 'handleSyncFiles');- expect(jobs.queueAll).toHaveBeenCalledWith([- expect.objectContaining({- name: JobName.SidecarCheck,- data: expect.objectContaining({ id: expect.any(String) }),- }),- ]);+ try {+ await ctx.scan(library.id);+ expect(syncFiles).toHaveBeenCalledTimes(3);+ } finally {+ walker.mockRestore();+ syncFiles.mockRestore();+ }+ await expect(ctx.getSidecarPaths(library.id)).resolves.toEqual(+ expected.toSorted((a, b) => a.originalPath.localeCompare(b.originalPath)),+ );});});});diff --git a/server/test/medium/specs/services/metadata.service.spec.ts b/server/test/medium/specs/services/metadata.service.spec.tsindex 7d26d230af..0c8bbaf152 100644--- a/server/test/medium/specs/services/metadata.service.spec.ts+++ b/server/test/medium/specs/services/metadata.service.spec.ts@@ -1,6 +1,6 @@import { Kysely } from 'kysely';import { Stats } from 'node:fs';-import { writeFile } from 'node:fs/promises';+import { mkdtempDisposable, writeFile } from 'node:fs/promises';import { tmpdir } from 'node:os';import { join } from 'node:path';import { AssetJobRepository } from 'src/repositories/asset-job.repository.js';@@ -30,7 +30,7 @@ type TimeZoneTest = {let defaultDatabase: Kysely<DB>;-const setup = (db?: Kysely<DB>) => {+const setup = (db?: Kysely<DB>, { realStorage = false } = {}) => {const { sut, ctx } = newMediumService(MetadataService, {database: db || defaultDatabase,real: [@@ -40,16 +40,19 @@ const setup = (db?: Kysely<DB>) => {MetadataRepository,SystemMetadataRepository,TagRepository,+ ...(realStorage ? [StorageRepository] : []),],- mock: [EventRepository, StorageRepository, LoggingRepository],+ mock: [EventRepository, LoggingRepository, ...(realStorage ? [] : [StorageRepository])],});- ctx.getMock(StorageRepository).stat.mockResolvedValue({- size: 123_456,- mtime: new Date(654_321),- mtimeMs: 654_321,- birthtimeMs: 654_322,- } as Stats);+ if (!realStorage) {+ ctx.getMock(StorageRepository).stat.mockResolvedValue({+ size: 123_456,+ mtime: new Date(654_321),+ mtimeMs: 654_321,+ birthtimeMs: 654_322,+ } as Stats);+ }return { sut, ctx };};@@ -77,6 +80,33 @@ describe(MetadataService.name, () => {expect(sut).toBeDefined();});+ describe('sidecar metadata extraction', () => {+ it('should persist metadata read from a discovered XMP sidecar', async () => {+ await using tempDir = await mkdtempDisposable(join(tmpdir(), 'immich-sidecar-'));+ const { sut, ctx } = setup(undefined, { realStorage: true });+ const originalPath = join(tempDir.path, 'photo.png');+ await writeFile(originalPath, newRandomImage());+ const metadata = ctx.get(MetadataRepository);+ await metadata.writeTags(originalPath, { Rating: 1 });+ await metadata.writeTags(`${originalPath}.xmp`, { Rating: 5, DateTimeOriginal: '2024:07:11 10:32:52+00:00' });+ const { user } = await ctx.newUser();+ const { asset } = await ctx.newAsset({ originalPath, ownerId: user.id });+ await ctx.newExif({ assetId: asset.id, description: '' });+ ctx.getMock(EventRepository).emit.mockResolvedValue();++ await sut.handleSidecarCheck({ id: asset.id });+ await sut.handleMetadataExtraction({ id: asset.id });++ await expect(+ ctx.database+ .selectFrom('asset_exif')+ .where('assetId', '=', asset.id)+ .select(['rating', 'dateTimeOriginal'])+ .executeTakeFirstOrThrow(),+ ).resolves.toEqual({ rating: 5, dateTimeOriginal: new Date('2024-07-11T10:32:52.000Z') });+ });+ });+describe('handleMetadataExtraction', () => {const timeZoneTests: TimeZoneTest[] = [{