Skip to content

High performance self-hosted photo and video management solution.

chore(server): better xmp testing (#31928)

Jonathan Jogenforsa010802431be
diff --git a/e2e/src/specs/server/api/asset.e2e-spec.ts b/e2e/src/specs/server/api/asset.e2e-spec.ts
index 6eb0dc4124..c29d105fb7 100644
--- a/e2e/src/specs/server/api/asset.e2e-spec.ts
+++ b/e2e/src/specs/server/api/asset.e2e-spec.ts
@@ -380,44 +380,6 @@ describe('/asset', () => {
expect(body).toMatchObject({ id: user1Assets[0].id, livePhotoVideoId: null });
});
- it.skip('should update date time original when sidecar file contains DateTimeOriginal', async () => {
- const sidecarData = `<?xpacket begin='?' id='W5M0MpCehiHzreSzNTczkc9d'?>
-<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='Image::ExifTool 12.40'>
-<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#'>
- <rdf:Description rdf:about=''
- xmlns:exif='http://ns.adobe.com/exif/1.0/'>
- <exif:ExifVersion>0220</exif:ExifVersion> <exif:DateTimeOriginal>2024-07-11T10:32:52Z</exif:DateTimeOriginal>
- <exif:GPSVersionID>2.3.0.0</exif:GPSVersionID>
- </rdf:Description>
-</rdf:RDF>
-</x:xmpmeta>
-<?xpacket end='w'?>`;
-
- const { id } = await utils.createAsset(user1.accessToken, {
- sidecarData: {
- bytes: Buffer.from(sidecarData),
- filename: 'example.xmp',
- },
- });
- await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
-
- const assetInfo = await utils.getAssetInfo(user1.accessToken, id);
- expect(assetInfo.exifInfo?.dateTimeOriginal).toBe('2024-07-11T10:32:52+00:00');
-
- const { status, body } = await request(app)
- .put(`/assets/${id}`)
- .set('Authorization', `Bearer ${user1.accessToken}`)
- .send({ dateTimeOriginal: '2023-11-19T18:11:00.000-07:00' });
-
- expect(body).toMatchObject({
- id,
- exifInfo: expect.objectContaining({
- dateTimeOriginal: '2023-11-20T01:11:00+00:00',
- }),
- });
- expect(status).toEqual(200);
- });
-
it('should update gps data', async () => {
const { status, body } = await request(app)
.put(`/assets/${user1Assets[0].id}`)
diff --git a/server/src/repositories/storage.repository.spec.ts b/server/src/repositories/storage.repository.spec.ts
index 3d1e503ed4..a6486532ce 100644
--- a/server/src/repositories/storage.repository.spec.ts
+++ b/server/src/repositories/storage.repository.spec.ts
@@ -1,4 +1,8 @@
import mockfs from 'mock-fs';
+import { R_OK } from 'node:constants';
+import { mkdtempDisposable, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
import { vitest } from 'vitest';
import { CrawlOptionsDto } from 'src/dtos/library.dto.js';
import { LoggingRepository } from 'src/repositories/logging.repository.js';
@@ -231,6 +235,52 @@ describe(StorageRepository.name, () => {
}
});
+ describe('walk', () => {
+ it.each([
+ { exclusionPatterns: [] },
+ { exclusionPatterns: ['**/*.xmp'] },
+ { exclusionPatterns: ['**/excluded/**'] },
+ ])('should only return assets and respect exclusions: $exclusionPatterns', async ({ exclusionPatterns }) => {
+ mockfs({
+ '/photos/photo.jpg': '',
+ '/photos/photo.nef': '',
+ '/photos/photo.jpg.xmp': '',
+ '/photos/photo.xmp': '',
+ '/photos/excluded/photo.jpg': '',
+ '/photos/excluded/photo.xmp': '',
+ });
+
+ const batches = await Array.fromAsync(sut.walk({ pathsToCrawl: ['/photos'], exclusionPatterns, take: 1 }));
+
+ expect(batches.every((batch) => batch.length === 1)).toBe(true);
+ expect(batches.flat().toSorted()).toEqual(
+ [
+ '/photos/photo.jpg',
+ '/photos/photo.nef',
+ ...(exclusionPatterns.includes('**/excluded/**') ? [] : ['/photos/excluded/photo.jpg']),
+ ].toSorted(),
+ );
+ });
+ });
+
+ describe('checkFileExists', () => {
+ it.for(['PHOTO.xmp', 'photo.XMP'])(
+ 'should not match %s with different case on a case-sensitive filesystem',
+ async (filename, { skip }) => {
+ await using tempDir = await mkdtempDisposable(join(tmpdir(), 'immich-storage-sidecar-'));
+ await writeFile(join(tempDir.path, 'case-probe'), 'test');
+ if (await sut.checkFileExists(join(tempDir.path, 'CASE-PROBE'), R_OK)) {
+ skip();
+ }
+ const candidate = join(tempDir.path, filename);
+ await writeFile(candidate, 'test');
+
+ await expect(sut.checkFileExists(candidate, R_OK)).resolves.toBe(true);
+ await expect(sut.checkFileExists(join(tempDir.path, 'photo.xmp'), R_OK)).resolves.toBe(false);
+ },
+ );
+ });
+
describe('watch', () => {
it('should register event handlers and close the watcher', async () => {
const onReady = vitest.fn();
diff --git a/server/src/services/job.service.spec.ts b/server/src/services/job.service.spec.ts
index baaf97cac7..b2dcc15b96 100644
--- a/server/src/services/job.service.spec.ts
+++ b/server/src/services/job.service.spec.ts
@@ -32,15 +32,32 @@ describe(JobService.name, () => {
expect(mocks.logger.error).not.toHaveBeenCalled();
});
- const tests: Array<{ item: JobItem; jobs: JobName[]; stub?: any }> = [
- {
- item: { name: JobName.SidecarCheck, data: { id: 'asset-1' } },
- jobs: [JobName.AssetExtractMetadata],
- },
- {
- item: { name: JobName.SidecarCheck, data: { id: 'asset-1' } },
- jobs: [JobName.AssetExtractMetadata],
+ it.each([JobStatus.Success, JobStatus.Skipped])(
+ 'should queue metadata extraction after a %s sidecar check and preserve its source',
+ async (status) => {
+ mocks.job.run.mockResolvedValue(status);
+ const job: JobItem = { name: JobName.SidecarCheck, data: { id: 'asset-1', source: 'upload' } };
+
+ await sut.onJobRun(QueueName.Sidecar, job);
+
+ expect(mocks.job.queue).toHaveBeenCalledExactlyOnceWith({
+ name: JobName.AssetExtractMetadata,
+ data: { id: 'asset-1', source: 'upload' },
+ });
+ expect(mocks.job.queueAll).not.toHaveBeenCalled();
},
+ );
+
+ it('should not queue metadata extraction after a failed sidecar check', async () => {
+ mocks.job.run.mockResolvedValue(JobStatus.Failed);
+
+ await sut.onJobRun(QueueName.Sidecar, { name: JobName.SidecarCheck, data: { id: 'asset-1', source: 'upload' } });
+
+ expect(mocks.job.queue).not.toHaveBeenCalled();
+ expect(mocks.job.queueAll).not.toHaveBeenCalled();
+ });
+
+ const tests: Array<{ item: JobItem; jobs: JobName[]; stub?: any }> = [
{
item: { name: JobName.StorageTemplateMigrationSingle, data: { id: 'asset-1', source: 'upload' } },
jobs: [JobName.AssetGenerateThumbnails],
diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.ts
index 53b3e7035c..2c281c7307 100644
--- a/server/src/services/library.service.spec.ts
+++ b/server/src/services/library.service.spec.ts
@@ -487,6 +487,7 @@ describe(LibraryService.name, () => {
await expect(sut.handleSyncAssets(mockAssetJob)).resolves.toBe(JobStatus.Success);
expect(mocks.asset.updateAll).not.toHaveBeenCalled();
+ expect(mocks.job.queueAll).not.toHaveBeenCalled();
});
it('should not touch fileCreatedAt when un-trashing an asset previously marked as offline', async () => {
diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts
index f8a2807427..31e2a427a8 100644
--- a/server/src/services/metadata.service.spec.ts
+++ b/server/src/services/metadata.service.spec.ts
@@ -1,5 +1,6 @@
import { BinaryField, ExifDateTime } from 'exiftool-vendored';
import { DateTime } from 'luxon';
+import { R_OK } from 'node:constants';
import { randomBytes } from 'node:crypto';
import { Stats } from 'node:fs';
import { defaults } from 'src/dtos/config.dto.js';
@@ -187,12 +188,12 @@ describe(MetadataService.name, () => {
expect(mocks.asset.update).not.toHaveBeenCalled();
});
- it('should handle a date in a sidecar file', async () => {
+ it.each(['CreationDate', 'DateTimeOriginal'] as const)('should prefer %s from a sidecar file', async (tag) => {
const originalDate = new Date('2023-11-21T16:13:17.517Z');
const sidecarDate = new Date('2022-01-01T00:00:00.000Z');
const asset = AssetFactory.from().file({ type: AssetFileType.Sidecar }).build();
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(getForMetadataExtraction(asset));
- mockReadTags({ CreationDate: originalDate.toISOString() }, { CreationDate: sidecarDate.toISOString() });
+ mockReadTags({ [tag]: originalDate.toISOString() }, { [tag]: sidecarDate.toISOString() });
await sut.handleMetadataExtraction({ id: asset.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
@@ -1920,6 +1921,7 @@ describe(MetadataService.name, () => {
type: AssetFileType.Sidecar,
path: '/path/to/IMG_123.jpg.xmp',
});
+ expect(mocks.storage.checkFileExists).toHaveBeenCalledExactlyOnceWith('/path/to/IMG_123.jpg.xmp', R_OK);
});
it('should detect a new sidecar at .xmp', async () => {
@@ -1939,34 +1941,163 @@ describe(MetadataService.name, () => {
type: AssetFileType.Sidecar,
path: '/path/to/IMG_123.xmp',
});
+ expect(mocks.storage.checkFileExists.mock.calls).toEqual([
+ ['/path/to/IMG_123.jpg.xmp', R_OK],
+ ['/path/to/IMG_123.xmp', R_OK],
+ ]);
});
- it('should unset sidecar path if file no longer exist', async () => {
+ it('should prefer the extended filename when both candidates exist', async () => {
+ const asset = forSidecarJob();
+ const candidates = new Set(['/path/to/IMG_123.jpg.xmp', '/path/to/IMG_123.xmp']);
+ mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
+ mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(candidates.has(path)));
+
+ await sut.handleSidecarCheck({ id: asset.id });
+
+ expect(mocks.asset.upsertFile).toHaveBeenCalledExactlyOnceWith({
+ assetId: asset.id,
+ type: AssetFileType.Sidecar,
+ path: '/path/to/IMG_123.jpg.xmp',
+ });
+ expect(mocks.storage.checkFileExists).toHaveBeenCalledExactlyOnceWith('/path/to/IMG_123.jpg.xmp', R_OK);
+ });
+
+ it('should preserve an existing association ahead of both conventional candidates', async () => {
+ const existing = '/elsewhere/custom.xmp';
const asset = forSidecarJob({
- originalPath: '/path/to/IMG_123.jpg',
- files: [{ id: 'sidecar', path: '/path/to/IMG_123.jpg.xmp', type: AssetFileType.Sidecar, isEdited: false }],
+ files: [{ id: 'sidecar', type: AssetFileType.Sidecar, path: existing, isEdited: false }],
});
+ const candidates = new Set([existing, '/path/to/IMG_123.jpg.xmp', '/path/to/IMG_123.xmp']);
+ mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
+ mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(candidates.has(path)));
+
+ await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Skipped);
+
+ expect(mocks.storage.checkFileExists).toHaveBeenCalledExactlyOnceWith(existing, R_OK);
+ expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
+ expect(mocks.asset.deleteFile).not.toHaveBeenCalled();
+ });
+
+ it.each(['/path/to/IMG_123.jpg.xmp', '/path/to/IMG_123.xmp'])(
+ 'should replace a missing association with %s',
+ async (replacement) => {
+ const existing = '/elsewhere/missing.xmp';
+ const asset = forSidecarJob({
+ files: [{ id: 'sidecar', type: AssetFileType.Sidecar, path: existing, isEdited: false }],
+ });
+ mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
+ mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === replacement));
+
+ await sut.handleSidecarCheck({ id: asset.id });
+
+ expect(mocks.storage.checkFileExists).toHaveBeenNthCalledWith(1, existing, R_OK);
+ expect(mocks.asset.upsertFile).toHaveBeenCalledExactlyOnceWith({
+ assetId: asset.id,
+ type: AssetFileType.Sidecar,
+ path: replacement,
+ });
+ expect(mocks.asset.deleteFile).not.toHaveBeenCalled();
+ },
+ );
+
+ it('should not create an association when no sidecar exists', async () => {
+ const asset = forSidecarJob();
mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
mocks.storage.checkFileExists.mockResolvedValue(false);
- await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Success);
+ await sut.handleSidecarCheck({ id: asset.id });
- expect(mocks.asset.deleteFile).toHaveBeenCalledWith({ assetId: asset.id, type: AssetFileType.Sidecar });
+ expect(mocks.storage.checkFileExists.mock.calls).toEqual([
+ ['/path/to/IMG_123.jpg.xmp', R_OK],
+ ['/path/to/IMG_123.xmp', R_OK],
+ ]);
+ expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
});
- it('should do nothing if the sidecar file still exists', async () => {
+ it.each(['/photos/Photo.JPG.xmp', '/photos/Photo.xmp'])(
+ 'should preserve filename case when discovering %s',
+ async (sidecarPath) => {
+ const asset = forSidecarJob({ originalPath: '/photos/Photo.JPG' });
+ mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
+ mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === sidecarPath));
+
+ await sut.handleSidecarCheck({ id: asset.id });
+
+ expect(mocks.asset.upsertFile).toHaveBeenCalledExactlyOnceWith({
+ assetId: asset.id,
+ type: AssetFileType.Sidecar,
+ path: sidecarPath,
+ });
+ },
+ );
+
+ it.each([
+ { filename: 'photo.edit.jpg', sidecar: 'photo.edit.xmp' },
+ { filename: '.photo.jpg', sidecar: '.photo.xmp' },
+ { filename: 'quoted"雪.jpg', sidecar: 'quoted"雪.xmp' },
+ ])('should preserve the basename when finding the fallback for $filename', async ({ filename, sidecar }) => {
+ const asset = forSidecarJob({ originalPath: `/photos/${filename}` });
+ const sidecarPath = `/photos/${sidecar}`;
+ mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
+ mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === sidecarPath));
+
+ await sut.handleSidecarCheck({ id: asset.id });
+
+ expect(mocks.asset.upsertFile).toHaveBeenCalledExactlyOnceWith({
+ assetId: asset.id,
+ type: AssetFileType.Sidecar,
+ path: sidecarPath,
+ });
+ });
+
+ it.each([false, true])(
+ 'should resolve shared JPG and RAW sidecars independently (extended JPG sidecar: %s)',
+ async (extended) => {
+ const jpg = forSidecarJob({ originalPath: '/photos/photo.jpg' });
+ const raw = forSidecarJob({ originalPath: '/photos/photo.nef' });
+ const shared = '/photos/photo.xmp';
+ const candidates = new Set([shared, ...(extended ? ['/photos/photo.jpg.xmp'] : [])]);
+ mocks.assetJob.getForSidecarCheckJob.mockResolvedValueOnce(jpg).mockResolvedValueOnce(raw);
+ mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(candidates.has(path)));
+
+ await sut.handleSidecarCheck({ id: jpg.id });
+ await sut.handleSidecarCheck({ id: raw.id });
+
+ expect(mocks.asset.upsertFile.mock.calls).toEqual([
+ [{ assetId: jpg.id, type: AssetFileType.Sidecar, path: extended ? '/photos/photo.jpg.xmp' : shared }],
+ [{ assetId: raw.id, type: AssetFileType.Sidecar, path: shared }],
+ ]);
+ },
+ );
+
+ it('should not match xmp files across directories', async () => {
+ const first = forSidecarJob({ originalPath: '/photos/first/photo.jpg' });
+ const second = forSidecarJob({ originalPath: '/photos/second/photo.jpg' });
+ mocks.assetJob.getForSidecarCheckJob.mockResolvedValueOnce(first).mockResolvedValueOnce(second);
+ mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === '/photos/second/photo.xmp'));
+
+ await sut.handleSidecarCheck({ id: first.id });
+ await sut.handleSidecarCheck({ id: second.id });
+
+ expect(mocks.asset.upsertFile).toHaveBeenCalledExactlyOnceWith({
+ assetId: second.id,
+ type: AssetFileType.Sidecar,
+ path: '/photos/second/photo.xmp',
+ });
+ });
+
+ it('should unset sidecar path if file no longer exist', async () => {
const asset = forSidecarJob({
originalPath: '/path/to/IMG_123.jpg',
files: [{ id: 'sidecar', path: '/path/to/IMG_123.jpg.xmp', type: AssetFileType.Sidecar, isEdited: false }],
});
-
mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
- mocks.storage.checkFileExists.mockResolvedValueOnce(true);
+ mocks.storage.checkFileExists.mockResolvedValue(false);
- await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Skipped);
+ await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Success);
- expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
- expect(mocks.asset.deleteFile).not.toHaveBeenCalled();
+ expect(mocks.asset.deleteFile).toHaveBeenCalledWith({ assetId: asset.id, type: AssetFileType.Sidecar });
});
});
diff --git a/server/test/medium/specs/repositories/asset.repository.spec.ts b/server/test/medium/specs/repositories/asset.repository.spec.ts
index 7a434d78df..11596da98b 100644
--- a/server/test/medium/specs/repositories/asset.repository.spec.ts
+++ b/server/test/medium/specs/repositories/asset.repository.spec.ts
@@ -1,5 +1,5 @@
import { Kysely } from 'kysely';
-import { AssetOrder, AssetOrderBy, AssetVisibility } from 'src/enum.js';
+import { AssetFileType, AssetOrder, AssetOrderBy, AssetVisibility } from 'src/enum.js';
import { AssetRepository } from 'src/repositories/asset.repository.js';
import { LoggingRepository } from 'src/repositories/logging.repository.js';
import { DB } from 'src/schema/index.js';
@@ -64,6 +64,31 @@ const keyframeRow = (assetId: string, n: number) => ({
});
describe(AssetRepository.name, () => {
+ describe('deleteFile', () => {
+ it('should delete only the requested asset and file type', async () => {
+ const { ctx, sut } = setup();
+ const { user } = await ctx.newUser();
+ const { asset } = await ctx.newAsset({ ownerId: user.id });
+ const { asset: other } = await ctx.newAsset({ ownerId: user.id });
+ await sut.upsertFiles([
+ { assetId: asset.id, type: AssetFileType.Sidecar, path: '/photos/photo.jpg.xmp' },
+ { assetId: asset.id, type: AssetFileType.Thumbnail, path: '/photos/thumbnail.jpg' },
+ { assetId: other.id, type: AssetFileType.Sidecar, path: '/photos/other.jpg.xmp' },
+ ]);
+
+ await sut.deleteFile({ assetId: asset.id, type: AssetFileType.Sidecar });
+
+ await expect(
+ ctx.database
+ .selectFrom('asset_file')
+ .where('assetId', 'in', [asset.id, other.id])
+ .select('path')
+ .orderBy('path')
+ .execute(),
+ ).resolves.toEqual([{ path: '/photos/other.jpg.xmp' }, { path: '/photos/thumbnail.jpg' }]);
+ });
+ });
+
describe('getTimeBucket', () => {
it('should order assets by local day first and fileCreatedAt within each day', async () => {
const { ctx, sut } = setup();
diff --git a/server/test/medium/specs/services/library.service.spec.ts b/server/test/medium/specs/services/library.service.spec.ts
index 42730db2d3..c250287984 100644
--- a/server/test/medium/specs/services/library.service.spec.ts
+++ b/server/test/medium/specs/services/library.service.spec.ts
@@ -1,11 +1,11 @@
import { Kysely } from 'kysely';
import { existsSync } from 'node:fs';
-import { copyFile, mkdir, mkdtemp, rm, utimes, writeFile } from 'node:fs/promises';
+import { mkdir, mkdtemp, rm, utimes, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import type { SystemConfig } from 'src/dtos/config.dto.js';
import { StorageCore } from 'src/cores/storage.core.js';
-import { AssetStatus, JobName, JobStatus } from 'src/enum.js';
+import { AssetFileType, AssetStatus, JobName, JobStatus } from 'src/enum.js';
import { AssetJobRepository } from 'src/repositories/asset-job.repository.js';
import { AssetRepository } from 'src/repositories/asset.repository.js';
import { CronRepository } from 'src/repositories/cron.repository.js';
@@ -18,8 +18,9 @@ import { LoggingRepository } from 'src/repositories/logging.repository.js';
import { StorageRepository } from 'src/repositories/storage.repository.js';
import { DB } from 'src/schema/index.js';
import { LibraryService } from 'src/services/library.service.js';
+import { MetadataService } from 'src/services/metadata.service.js';
import { systemConfigStub } from 'test/fixtures/system-config.stub.js';
-import { MediumTestContext, testAssetsDir } from 'test/medium.factory.js';
+import { MediumTestContext, newMediumService } from 'test/medium.factory.js';
import { newUuid } from 'test/small.factory.js';
import { getKyselyDB } from 'test/utils.js';
@@ -37,14 +38,9 @@ const createFile = async (filePath: string, modifiedAt: Date = fileModifiedAt) =
return filePath;
};
-const copyTestAsset = async (source: string, filePath: string, modifiedAt: Date = fileModifiedAt) => {
- await mkdir(dirname(filePath), { recursive: true });
- await copyFile(join(testAssetsDir, source), filePath);
- await utimes(filePath, modifiedAt, modifiedAt);
- return filePath;
-};
-
class LibraryTestContext extends MediumTestContext<typeof LibraryService> {
+ private metadataService: MetadataService;
+
constructor(database: Kysely<DB>) {
super(LibraryService, {
database,
@@ -57,6 +53,12 @@ class LibraryTestContext extends MediumTestContext<typeof LibraryService> {
jobs.queueAll.mockResolvedValue();
this.getMock(EventRepository).emit.mockResolvedValue();
+
+ this.metadataService = newMediumService(MetadataService, {
+ database,
+ real: [AssetRepository, AssetJobRepository, StorageRepository],
+ mock: [LoggingRepository],
+ }).sut;
}
async createLibrary(options: { importPaths?: string[]; exclusionPatterns?: string[] } = {}) {
@@ -74,20 +76,45 @@ class LibraryTestContext extends MediumTestContext<typeof LibraryService> {
const jobs = this.getMock(JobRepository);
jobs.queue.mockClear();
+ jobs.queueAll.mockClear();
await this.sut.handleQueueSyncFiles({ id: libraryId });
for (const [job] of jobs.queue.mock.calls) {
if (job.name === JobName.LibrarySyncFiles) {
await this.sut.handleSyncFiles(job.data);
}
}
+ await this.discoverSidecars();
jobs.queue.mockClear();
+ jobs.queueAll.mockClear();
await this.sut.handleQueueSyncAssets({ id: libraryId });
for (const [job] of jobs.queue.mock.calls) {
if (job.name === JobName.LibrarySyncAssets) {
await this.sut.handleSyncAssets(job.data);
}
}
+ await this.discoverSidecars();
+ }
+
+ private async discoverSidecars() {
+ for (const [jobs] of this.getMock(JobRepository).queueAll.mock.calls) {
+ for (const job of jobs) {
+ if (job.name === JobName.SidecarCheck) {
+ await this.metadataService.handleSidecarCheck(job.data);
+ }
+ }
+ }
+ }
+
+ getSidecarPaths(libraryId: string) {
+ return this.database
+ .selectFrom('asset')
+ .innerJoin('asset_file', 'asset_file.assetId', 'asset.id')
+ .where('libraryId', '=', libraryId)
+ .where('asset_file.type', '=', AssetFileType.Sidecar)
+ .select(['originalPath', 'asset_file.path as sidecarPath'])
+ .orderBy('originalPath')
+ .execute();
}
/** The paths a library scan left visible, i.e. neither offline nor trashed */
@@ -813,95 +840,38 @@ describe(LibraryService.name, () => {
expect(updated).toEqual(expect.objectContaining({ isOffline: false }));
expect(updated?.deletedAt).toBeInstanceOf(Date);
});
-
- it('should queue sidecar checks for assets whose file changed', async () => {
- const { sut, ctx } = setup();
- const jobs = ctx.getMock(JobRepository);
- const library = await ctx.createLibrary({ importPaths: [importPath] });
- const rawPath = await copyTestAsset('formats/raw/Nikon/D80/glarus.nef', join(importPath, 'glarus.nef'));
-
- const { asset } = await ctx.newAsset({
- ownerId: library.ownerId,
- libraryId: library.id,
- originalPath: rawPath,
- // the file on disk has a newer modified time
- fileModifiedAt: new Date(fileModifiedAt.valueOf() - 1000),
- isExternal: true,
- isOffline: false,
- status: AssetStatus.Active,
- });
-
- await expect(
- sut.handleSyncAssets({
- libraryId: library.id,
- importPaths: library.importPaths,
- exclusionPatterns: library.exclusionPatterns,
- assetIds: [asset.id],
- progressCounter: 1,
- totalAssets: 1,
- }),
- ).resolves.toBe(JobStatus.Success);
-
- expect(jobs.queueAll).toHaveBeenCalledWith([
- {
- name: JobName.SidecarCheck,
- data: { id: asset.id, source: 'upload' },
- },
- ]);
- });
-
- it('should not queue sidecar checks for unchanged assets', async () => {
- const { sut, ctx } = setup();
- const jobs = ctx.getMock(JobRepository);
- const library = await ctx.createLibrary({ importPaths: [importPath] });
- const rawPath = await copyTestAsset('formats/raw/Nikon/D80/glarus.nef', join(importPath, 'glarus.nef'));
-
- const { asset } = await ctx.newAsset({
- ownerId: library.ownerId,
- libraryId: library.id,
- originalPath: rawPath,
- fileModifiedAt,
- isExternal: true,
- isOffline: false,
- status: AssetStatus.Active,
- });
-
- await expect(
- sut.handleSyncAssets({
- libraryId: library.id,
- importPaths: library.importPaths,
- exclusionPatterns: library.exclusionPatterns,
- assetIds: [asset.id],
- progressCounter: 1,
- totalAssets: 1,
- }),
- ).resolves.toBe(JobStatus.Success);
-
- expect(jobs.queueAll).not.toHaveBeenCalled();
- });
});
- describe('handleSyncFiles', () => {
- it('should queue sidecar checks for newly imported assets', async () => {
- const { sut, ctx } = setup();
- const jobs = ctx.getMock(JobRepository);
- const library = await ctx.createLibrary({ importPaths: [importPath] });
- const rawPath = await copyTestAsset('formats/raw/Nikon/D80/glarus.nef', join(importPath, 'glarus.nef'));
-
- await expect(
- sut.handleSyncFiles({
- libraryId: library.id,
- paths: [rawPath],
- progressCounter: 1,
- }),
- ).resolves.toBe(JobStatus.Success);
+ describe('sidecar discovery during library scans', () => {
+ it('should only associate with sidecars in the same directory across multiple scan batches and import paths', async () => {
+ const { ctx } = setup();
+ const secondImportPath = join(importRoot, 'second');
+ const library = await ctx.createLibrary({
+ importPaths: [importPath, secondImportPath],
+ });
+ const expected = [];
+ for (const directory of [importPath, secondImportPath]) {
+ for (const name of ['a', 'b', 'c']) {
+ const originalPath = await createFile(join(directory, name, 'photo.jpg'));
+ const sidecarPath = await createFile(join(directory, name, 'photo.xmp'));
+ expected.push({ originalPath, sidecarPath });
+ }
+ }
+ const storage = ctx.get(StorageRepository);
+ const walk = storage.walk.bind(storage);
+ const walker = vi.spyOn(storage, 'walk').mockImplementation((options) => walk({ ...options, take: 2 }));
+ const syncFiles = vi.spyOn(ctx.sut, 'handleSyncFiles');
- expect(jobs.queueAll).toHaveBeenCalledWith([
- expect.objectContaining({
- name: JobName.SidecarCheck,
- data: expect.objectContaining({ id: expect.any(String) }),
- }),
- ]);
+ try {
+ await ctx.scan(library.id);
+ expect(syncFiles).toHaveBeenCalledTimes(3);
+ } finally {
+ walker.mockRestore();
+ syncFiles.mockRestore();
+ }
+ await expect(ctx.getSidecarPaths(library.id)).resolves.toEqual(
+ expected.toSorted((a, b) => a.originalPath.localeCompare(b.originalPath)),
+ );
});
});
});
diff --git a/server/test/medium/specs/services/metadata.service.spec.ts b/server/test/medium/specs/services/metadata.service.spec.ts
index 7d26d230af..0c8bbaf152 100644
--- a/server/test/medium/specs/services/metadata.service.spec.ts
+++ b/server/test/medium/specs/services/metadata.service.spec.ts
@@ -1,6 +1,6 @@
import { Kysely } from 'kysely';
import { Stats } from 'node:fs';
-import { writeFile } from 'node:fs/promises';
+import { mkdtempDisposable, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { AssetJobRepository } from 'src/repositories/asset-job.repository.js';
@@ -30,7 +30,7 @@ type TimeZoneTest = {
let defaultDatabase: Kysely<DB>;
-const setup = (db?: Kysely<DB>) => {
+const setup = (db?: Kysely<DB>, { realStorage = false } = {}) => {
const { sut, ctx } = newMediumService(MetadataService, {
database: db || defaultDatabase,
real: [
@@ -40,16 +40,19 @@ const setup = (db?: Kysely<DB>) => {
MetadataRepository,
SystemMetadataRepository,
TagRepository,
+ ...(realStorage ? [StorageRepository] : []),
],
- mock: [EventRepository, StorageRepository, LoggingRepository],
+ mock: [EventRepository, LoggingRepository, ...(realStorage ? [] : [StorageRepository])],
});
- ctx.getMock(StorageRepository).stat.mockResolvedValue({
- size: 123_456,
- mtime: new Date(654_321),
- mtimeMs: 654_321,
- birthtimeMs: 654_322,
- } as Stats);
+ if (!realStorage) {
+ ctx.getMock(StorageRepository).stat.mockResolvedValue({
+ size: 123_456,
+ mtime: new Date(654_321),
+ mtimeMs: 654_321,
+ birthtimeMs: 654_322,
+ } as Stats);
+ }
return { sut, ctx };
};
@@ -77,6 +80,33 @@ describe(MetadataService.name, () => {
expect(sut).toBeDefined();
});
+ describe('sidecar metadata extraction', () => {
+ it('should persist metadata read from a discovered XMP sidecar', async () => {
+ await using tempDir = await mkdtempDisposable(join(tmpdir(), 'immich-sidecar-'));
+ const { sut, ctx } = setup(undefined, { realStorage: true });
+ const originalPath = join(tempDir.path, 'photo.png');
+ await writeFile(originalPath, newRandomImage());
+ const metadata = ctx.get(MetadataRepository);
+ await metadata.writeTags(originalPath, { Rating: 1 });
+ await metadata.writeTags(`${originalPath}.xmp`, { Rating: 5, DateTimeOriginal: '2024:07:11 10:32:52+00:00' });
+ const { user } = await ctx.newUser();
+ const { asset } = await ctx.newAsset({ originalPath, ownerId: user.id });
+ await ctx.newExif({ assetId: asset.id, description: '' });
+ ctx.getMock(EventRepository).emit.mockResolvedValue();
+
+ await sut.handleSidecarCheck({ id: asset.id });
+ await sut.handleMetadataExtraction({ id: asset.id });
+
+ await expect(
+ ctx.database
+ .selectFrom('asset_exif')
+ .where('assetId', '=', asset.id)
+ .select(['rating', 'dateTimeOriginal'])
+ .executeTakeFirstOrThrow(),
+ ).resolves.toEqual({ rating: 5, dateTimeOriginal: new Date('2024-07-11T10:32:52.000Z') });
+ });
+ });
+
describe('handleMetadataExtraction', () => {
const timeZoneTests: TimeZoneTest[] = [
{