Skip to content

High performance self-hosted photo and video management solution.

fix: asset update race condition (#24384)

Daniel Dietzlerc15998e80595
diff --git a/server/src/database.ts b/server/src/database.ts
index a3c38ae61e..9f4494b720 100644
--- a/server/src/database.ts
+++ b/server/src/database.ts
@@ -240,7 +240,7 @@ export type Session = {
isPendingSyncReset: boolean;
};
-export type Exif = Omit<Selectable<AssetExifTable>, 'updatedAt' | 'updateId'>;
+export type Exif = Omit<Selectable<AssetExifTable>, 'updatedAt' | 'updateId' | 'lockedProperties'>;
export type Person = {
createdAt: Date;
@@ -465,3 +465,13 @@ export const columns = {
'plugin.updatedAt as updatedAt',
],
} as const;
+
+export type LockableProperty = (typeof lockableProperties)[number];
+export const lockableProperties = [
+ 'description',
+ 'dateTimeOriginal',
+ 'latitude',
+ 'longitude',
+ 'rating',
+ 'timeZone',
+] as const;
diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sql
index d1b4e5a72e..b736998e28 100644
--- a/server/src/queries/asset.job.repository.sql
+++ b/server/src/queries/asset.job.repository.sql
@@ -50,9 +50,11 @@ select
where
"asset"."id" = "tag_asset"."assetId"
) as agg
- ) as "tags"
+ ) as "tags",
+ to_json("asset_exif") as "exifInfo"
from
"asset"
+ inner join "asset_exif" on "asset"."id" = "asset_exif"."assetId"
where
"asset"."id" = $2::uuid
limit
@@ -224,6 +226,14 @@ from
where
"asset"."id" = $2
+-- AssetJobRepository.getLockedPropertiesForMetadataExtraction
+select
+ "asset_exif"."lockedProperties"
+from
+ "asset_exif"
+where
+ "asset_exif"."assetId" = $1
+
-- AssetJobRepository.getAlbumThumbnailFiles
select
"asset_file"."id",
diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql
index 01cc6a7a89..f25a0798d2 100644
--- a/server/src/queries/asset.repository.sql
+++ b/server/src/queries/asset.repository.sql
@@ -1,19 +1,49 @@
-- NOTE: This file is auto generated by ./sql-generator
+-- AssetRepository.upsertExif
+insert into
+ "asset_exif" ("dateTimeOriginal", "lockedProperties")
+values
+ ($1, $2)
+on conflict ("assetId") do update
+set
+ "dateTimeOriginal" = "excluded"."dateTimeOriginal",
+ "lockedProperties" = nullif(
+ array(
+ select distinct
+ unnest("asset_exif"."lockedProperties" || $3)
+ ),
+ '{}'
+ )
+
-- AssetRepository.updateAllExif
update "asset_exif"
set
- "model" = $1
+ "model" = $1,
+ "lockedProperties" = nullif(
+ array(
+ select distinct
+ unnest("asset_exif"."lockedProperties" || $2)
+ ),
+ '{}'
+ )
where
- "assetId" in ($2)
+ "assetId" in ($3)
-- AssetRepository.updateDateTimeOriginal
update "asset_exif"
set
"dateTimeOriginal" = "dateTimeOriginal" + $1::interval,
- "timeZone" = $2
+ "timeZone" = $2,
+ "lockedProperties" = nullif(
+ array(
+ select distinct
+ unnest("asset_exif"."lockedProperties" || $3)
+ ),
+ '{}'
+ )
where
- "assetId" in ($3)
+ "assetId" in ($4)
returning
"assetId",
"dateTimeOriginal",
diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts
index de994b08cd..214d42747f 100644
--- a/server/src/repositories/asset-job.repository.ts
+++ b/server/src/repositories/asset-job.repository.ts
@@ -50,6 +50,7 @@ export class AssetJobRepository {
.whereRef('asset.id', '=', 'tag_asset.assetId'),
).as('tags'),
)
+ .$call(withExifInner)
.limit(1)
.executeTakeFirst();
}
@@ -128,6 +129,16 @@ export class AssetJobRepository {
.executeTakeFirst();
}
+ @GenerateSql({ params: [DummyValue.UUID] })
+ async getLockedPropertiesForMetadataExtraction(assetId: string) {
+ return this.db
+ .selectFrom('asset_exif')
+ .select('asset_exif.lockedProperties')
+ .where('asset_exif.assetId', '=', assetId)
+ .executeTakeFirst()
+ .then((row) => row?.lockedProperties ?? []);
+ }
+
@GenerateSql({ params: [DummyValue.UUID, AssetFileType.Thumbnail] })
getAlbumThumbnailFiles(id: string, fileType?: AssetFileType) {
return this.db
diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts
index 4b8cbd7a7a..7db3a76f12 100644
--- a/server/src/repositories/asset.repository.ts
+++ b/server/src/repositories/asset.repository.ts
@@ -1,8 +1,8 @@
import { Injectable } from '@nestjs/common';
-import { Insertable, Kysely, NotNull, Selectable, sql, Updateable, UpdateResult } from 'kysely';
+import { ExpressionBuilder, Insertable, Kysely, NotNull, Selectable, sql, Updateable, UpdateResult } from 'kysely';
import { isEmpty, isUndefined, omitBy } from 'lodash';
import { InjectKysely } from 'nestjs-kysely';
-import { Stack } from 'src/database';
+import { LockableProperty, Stack } from 'src/database';
import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
import { AuthDto } from 'src/dtos/auth.dto';
import { AssetFileType, AssetMetadataKey, AssetOrder, AssetStatus, AssetType, AssetVisibility } from 'src/enum';
@@ -113,51 +113,77 @@ interface GetByIdsRelations {
tags?: boolean;
}
+const distinctLocked = <T extends LockableProperty[] | null>(eb: ExpressionBuilder<DB, 'asset_exif'>, columns: T) =>
+ sql<T>`nullif(array(select distinct unnest(${eb.ref('asset_exif.lockedProperties')} || ${columns})), '{}')`;
+
@Injectable()
export class AssetRepository {
constructor(@InjectKysely() private db: Kysely<DB>) {}
- async upsertExif(exif: Insertable<AssetExifTable>): Promise<void> {
- const value = { ...exif, assetId: asUuid(exif.assetId) };
+ @GenerateSql({
+ params: [
+ { dateTimeOriginal: DummyValue.DATE, lockedProperties: ['dateTimeOriginal'] },
+ { lockedPropertiesBehavior: 'append' },
+ ],
+ })
+ async upsertExif(
+ exif: Insertable<AssetExifTable>,
+ { lockedPropertiesBehavior }: { lockedPropertiesBehavior: 'override' | 'append' | 'skip' },
+ ): Promise<void> {
await this.db
.insertInto('asset_exif')
- .values(value)
+ .values(exif)
.onConflict((oc) =>
- oc.column('assetId').doUpdateSet((eb) =>
- removeUndefinedKeys(
- {
- description: eb.ref('excluded.description'),
- exifImageWidth: eb.ref('excluded.exifImageWidth'),
- exifImageHeight: eb.ref('excluded.exifImageHeight'),
- fileSizeInByte: eb.ref('excluded.fileSizeInByte'),
- orientation: eb.ref('excluded.orientation'),
- dateTimeOriginal: eb.ref('excluded.dateTimeOriginal'),
- modifyDate: eb.ref('excluded.modifyDate'),
- timeZone: eb.ref('excluded.timeZone'),
- latitude: eb.ref('excluded.latitude'),
- longitude: eb.ref('excluded.longitude'),
- projectionType: eb.ref('excluded.projectionType'),
- city: eb.ref('excluded.city'),
- livePhotoCID: eb.ref('excluded.livePhotoCID'),
- autoStackId: eb.ref('excluded.autoStackId'),
- state: eb.ref('excluded.state'),
- country: eb.ref('excluded.country'),
- make: eb.ref('excluded.make'),
- model: eb.ref('excluded.model'),
- lensModel: eb.ref('excluded.lensModel'),
- fNumber: eb.ref('excluded.fNumber'),
- focalLength: eb.ref('excluded.focalLength'),
- iso: eb.ref('excluded.iso'),
- exposureTime: eb.ref('excluded.exposureTime'),
- profileDescription: eb.ref('excluded.profileDescription'),
- colorspace: eb.ref('excluded.colorspace'),
- bitsPerSample: eb.ref('excluded.bitsPerSample'),
- rating: eb.ref('excluded.rating'),
- fps: eb.ref('excluded.fps'),
- },
- value,
- ),
- ),
+ oc.column('assetId').doUpdateSet((eb) => {
+ const updateLocked = <T extends keyof AssetExifTable>(col: T) => eb.ref(`excluded.${col}`);
+ const skipLocked = <T extends keyof AssetExifTable>(col: T) =>
+ eb
+ .case()
+ .when(sql`${col}`, '=', eb.fn.any('asset_exif.lockedProperties'))
+ .then(eb.ref(`asset_exif.${col}`))
+ .else(eb.ref(`excluded.${col}`))
+ .end();
+ const ref = lockedPropertiesBehavior === 'skip' ? skipLocked : updateLocked;
+ return {
+ ...removeUndefinedKeys(
+ {
+ description: ref('description'),
+ exifImageWidth: ref('exifImageWidth'),
+ exifImageHeight: ref('exifImageHeight'),
+ fileSizeInByte: ref('fileSizeInByte'),
+ orientation: ref('orientation'),
+ dateTimeOriginal: ref('dateTimeOriginal'),
+ modifyDate: ref('modifyDate'),
+ timeZone: ref('timeZone'),
+ latitude: ref('latitude'),
+ longitude: ref('longitude'),
+ projectionType: ref('projectionType'),
+ city: ref('city'),
+ livePhotoCID: ref('livePhotoCID'),
+ autoStackId: ref('autoStackId'),
+ state: ref('state'),
+ country: ref('country'),
+ make: ref('make'),
+ model: ref('model'),
+ lensModel: ref('lensModel'),
+ fNumber: ref('fNumber'),
+ focalLength: ref('focalLength'),
+ iso: ref('iso'),
+ exposureTime: ref('exposureTime'),
+ profileDescription: ref('profileDescription'),
+ colorspace: ref('colorspace'),
+ bitsPerSample: ref('bitsPerSample'),
+ rating: ref('rating'),
+ fps: ref('fps'),
+ lockedProperties:
+ lockedPropertiesBehavior === 'append'
+ ? distinctLocked(eb, exif.lockedProperties ?? null)
+ : ref('lockedProperties'),
+ },
+ exif,
+ ),
+ };
+ }),
)
.execute();
}
@@ -169,19 +195,26 @@ export class AssetRepository {
return;
}
- await this.db.updateTable('asset_exif').set(options).where('assetId', 'in', ids).execute();
+ await this.db
+ .updateTable('asset_exif')
+ .set((eb) => ({
+ ...options,
+ lockedProperties: distinctLocked(eb, Object.keys(options) as LockableProperty[]),
+ }))
+ .where('assetId', 'in', ids)
+ .execute();
}
@GenerateSql({ params: [[DummyValue.UUID], DummyValue.NUMBER, DummyValue.STRING] })
@Chunked()
- async updateDateTimeOriginal(
- ids: string[],
- delta?: number,
- timeZone?: string,
- ): Promise<{ assetId: string; dateTimeOriginal: Date | null; timeZone: string | null }[]> {
- return await this.db
+ updateDateTimeOriginal(ids: string[], delta?: number, timeZone?: string) {
+ return this.db
.updateTable('asset_exif')
- .set({ dateTimeOriginal: sql`"dateTimeOriginal" + ${(delta ?? 0) + ' minute'}::interval`, timeZone })
+ .set((eb) => ({
+ dateTimeOriginal: sql`"dateTimeOriginal" + ${(delta ?? 0) + ' minute'}::interval`,
+ timeZone,
+ lockedProperties: distinctLocked(eb, ['dateTimeOriginal', 'timeZone']),
+ }))
.where('assetId', 'in', ids)
.returning(['assetId', 'dateTimeOriginal', 'timeZone'])
.execute();
diff --git a/server/src/schema/migrations/1764957138636-AddLockedPropertiesToAssetExif.ts b/server/src/schema/migrations/1764957138636-AddLockedPropertiesToAssetExif.ts
new file mode 100644
index 0000000000..0cd024a4d8
--- /dev/null
+++ b/server/src/schema/migrations/1764957138636-AddLockedPropertiesToAssetExif.ts
@@ -0,0 +1,9 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+ await sql`ALTER TABLE "asset_exif" ADD "lockedProperties" character varying[];`.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+ await sql`ALTER TABLE "asset_exif" DROP COLUMN "lockedProperties";`.execute(db);
+}
diff --git a/server/src/schema/tables/asset-exif.table.ts b/server/src/schema/tables/asset-exif.table.ts
index f28735a2db..346098a72c 100644
--- a/server/src/schema/tables/asset-exif.table.ts
+++ b/server/src/schema/tables/asset-exif.table.ts
@@ -1,3 +1,4 @@
+import { LockableProperty } from 'src/database';
import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators';
import { AssetTable } from 'src/schema/tables/asset.table';
import { Column, ForeignKeyColumn, Generated, Int8, Table, Timestamp, UpdateDateColumn } from 'src/sql-tools';
@@ -97,4 +98,7 @@ export class AssetExifTable {
@UpdateIdColumn({ index: true })
updateId!: Generated<string>;
+
+ @Column({ type: 'character varying', array: true, nullable: true })
+ lockedProperties!: Array<LockableProperty> | null;
}
diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts
index d2e1c14210..2bb8530c1c 100644
--- a/server/src/services/asset-media.service.ts
+++ b/server/src/services/asset-media.service.ts
@@ -370,7 +370,10 @@ export class AssetMediaService extends BaseService {
: this.assetRepository.deleteFile({ assetId, type: AssetFileType.Sidecar }));
await this.storageRepository.utimes(file.originalPath, new Date(), new Date(dto.fileModifiedAt));
- await this.assetRepository.upsertExif({ assetId, fileSizeInByte: file.size });
+ await this.assetRepository.upsertExif(
+ { assetId, fileSizeInByte: file.size },
+ { lockedPropertiesBehavior: 'override' },
+ );
await this.jobRepository.queue({
name: JobName.AssetExtractMetadata,
data: { id: assetId, source: 'upload' },
@@ -399,7 +402,10 @@ export class AssetMediaService extends BaseService {
});
const { size } = await this.storageRepository.stat(created.originalPath);
- await this.assetRepository.upsertExif({ assetId: created.id, fileSizeInByte: size });
+ await this.assetRepository.upsertExif(
+ { assetId: created.id, fileSizeInByte: size },
+ { lockedPropertiesBehavior: 'override' },
+ );
await this.jobRepository.queue({ name: JobName.AssetExtractMetadata, data: { id: created.id, source: 'copy' } });
return created;
}
@@ -440,7 +446,10 @@ export class AssetMediaService extends BaseService {
await this.storageRepository.utimes(sidecarFile.originalPath, new Date(), new Date(dto.fileModifiedAt));
}
await this.storageRepository.utimes(file.originalPath, new Date(), new Date(dto.fileModifiedAt));
- await this.assetRepository.upsertExif({ assetId: asset.id, fileSizeInByte: file.size });
+ await this.assetRepository.upsertExif(
+ { assetId: asset.id, fileSizeInByte: file.size },
+ { lockedPropertiesBehavior: 'override' },
+ );
await this.eventRepository.emit('AssetCreate', { asset });
diff --git a/server/src/services/asset.service.spec.ts b/server/src/services/asset.service.spec.ts
index 878721e0a7..5e1cce2ccf 100755
--- a/server/src/services/asset.service.spec.ts
+++ b/server/src/services/asset.service.spec.ts
@@ -225,7 +225,10 @@ describe(AssetService.name, () => {
await sut.update(authStub.admin, 'asset-1', { description: 'Test description' });
- expect(mocks.asset.upsertExif).toHaveBeenCalledWith({ assetId: 'asset-1', description: 'Test description' });
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+ { assetId: 'asset-1', description: 'Test description', lockedProperties: ['description'] },
+ { lockedPropertiesBehavior: 'append' },
+ );
});
it('should update the exif rating', async () => {
@@ -235,7 +238,14 @@ describe(AssetService.name, () => {
await sut.update(authStub.admin, 'asset-1', { rating: 3 });
- expect(mocks.asset.upsertExif).toHaveBeenCalledWith({ assetId: 'asset-1', rating: 3 });
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+ {
+ assetId: 'asset-1',
+ rating: 3,
+ lockedProperties: ['rating'],
+ },
+ { lockedPropertiesBehavior: 'append' },
+ );
});
it('should fail linking a live video if the motion part could not be found', async () => {
@@ -427,9 +437,7 @@ describe(AssetService.name, () => {
});
expect(mocks.asset.updateAll).toHaveBeenCalled();
expect(mocks.asset.updateAllExif).toHaveBeenCalledWith(['asset-1'], { latitude: 0, longitude: 0 });
- expect(mocks.job.queueAll).toHaveBeenCalledWith([
- { name: JobName.SidecarWrite, data: { id: 'asset-1', latitude: 0, longitude: 0 } },
- ]);
+ expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.SidecarWrite, data: { id: 'asset-1' } }]);
});
it('should update exif table if latitude field is provided', async () => {
@@ -450,9 +458,7 @@ describe(AssetService.name, () => {
latitude: 30,
longitude: 50,
});
- expect(mocks.job.queueAll).toHaveBeenCalledWith([
- { name: JobName.SidecarWrite, data: { id: 'asset-1', dateTimeOriginal, latitude: 30, longitude: 50 } },
- ]);
+ expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.SidecarWrite, data: { id: 'asset-1' } }]);
});
it('should update Assets table if duplicateId is provided as null', async () => {
@@ -482,18 +488,7 @@ describe(AssetService.name, () => {
timeZone,
});
expect(mocks.asset.updateDateTimeOriginal).toHaveBeenCalledWith(['asset-1'], dateTimeRelative, timeZone);
- expect(mocks.job.queueAll).toHaveBeenCalledWith([
- {
- name: JobName.SidecarWrite,
- data: {
- id: 'asset-1',
- dateTimeOriginal: '2020-02-25T06:41:00.000+02:00',
- description: undefined,
- latitude: undefined,
- longitude: undefined,
- },
- },
- ]);
+ expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.SidecarWrite, data: { id: 'asset-1' } }]);
});
});
diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.ts
index 32c6526394..282d74a9b1 100644
--- a/server/src/services/asset.service.ts
+++ b/server/src/services/asset.service.ts
@@ -30,9 +30,10 @@ import {
QueueName,
} from 'src/enum';
import { BaseService } from 'src/services/base.service';
-import { ISidecarWriteJob, JobItem, JobOf } from 'src/types';
+import { JobItem, JobOf } from 'src/types';
import { requireElevatedPermission } from 'src/utils/access';
import { getAssetFiles, getMyPartnerIds, onAfterUnlink, onBeforeLink, onBeforeUnlink } from 'src/utils/asset.util';
+import { updateLockedColumns } from 'src/utils/database';
@Injectable()
export class AssetService extends BaseService {
@@ -142,56 +143,26 @@ export class AssetService extends BaseService {
} = dto;
await this.requireAccess({ auth, permission: Permission.AssetUpdate, ids });
- const assetDto = { isFavorite, visibility, duplicateId };
- const exifDto = { latitude, longitude, rating, description, dateTimeOriginal };
+ const assetDto = _.omitBy({ isFavorite, visibility, duplicateId }, _.isUndefined);
+ const exifDto = _.omitBy({ latitude, longitude, rating, description, dateTimeOriginal }, _.isUndefined);
- const isExifChanged = Object.values(exifDto).some((v) => v !== undefined);
- if (isExifChanged) {
+ if (Object.keys(exifDto).length > 0) {
await this.assetRepository.updateAllExif(ids, exifDto);
}
- const assets =
- (dateTimeRelative !== undefined && dateTimeRelative !== 0) || timeZone !== undefined
- ? await this.assetRepository.updateDateTimeOriginal(ids, dateTimeRelative, timeZone)
- : undefined;
-
- const dateTimesWithTimezone = assets
- ? assets.map((asset) => {
- const isoString = asset.dateTimeOriginal?.toISOString();
- let dateTime = isoString ? DateTime.fromISO(isoString) : null;
-
- if (dateTime && asset.timeZone) {
- dateTime = dateTime.setZone(asset.timeZone);
- }
-
- return {
- assetId: asset.assetId,
- dateTimeOriginal: dateTime?.toISO() ?? null,
- };
- })
- : ids.map((id) => ({ assetId: id, dateTimeOriginal }));
-
- if (dateTimesWithTimezone.length > 0) {
- await this.jobRepository.queueAll(
- dateTimesWithTimezone.map(({ assetId: id, dateTimeOriginal }) => ({
- name: JobName.SidecarWrite,
- data: {
- ...exifDto,
- id,
- dateTimeOriginal: dateTimeOriginal ?? undefined,
- },
- })),
- );
+ if ((dateTimeRelative !== undefined && dateTimeRelative !== 0) || timeZone !== undefined) {
+ await this.assetRepository.updateDateTimeOriginal(ids, dateTimeRelative, timeZone);
}
- const isAssetChanged = Object.values(assetDto).some((v) => v !== undefined);
- if (isAssetChanged) {
+ if (Object.keys(assetDto).length > 0) {
await this.assetRepository.updateAll(ids, assetDto);
+ }
- if (visibility === AssetVisibility.Locked) {
- await this.albumRepository.removeAssetsFromAll(ids);
- }
+ if (visibility === AssetVisibility.Locked) {
+ await this.albumRepository.removeAssetsFromAll(ids);
}
+
+ await this.jobRepository.queueAll(ids.map((id) => ({ name: JobName.SidecarWrite, data: { id } })));
}
async copy(
@@ -456,12 +427,25 @@ export class AssetService extends BaseService {
return asset;
}
- private async updateExif(dto: ISidecarWriteJob) {
+ private async updateExif(dto: {
+ id: string;
+ description?: string;
+ dateTimeOriginal?: string;
+ latitude?: number;
+ longitude?: number;
+ rating?: number;
+ }) {
const { id, description, dateTimeOriginal, latitude, longitude, rating } = dto;
const writes = _.omitBy({ description, dateTimeOriginal, latitude, longitude, rating }, _.isUndefined);
if (Object.keys(writes).length > 0) {
- await this.assetRepository.upsertExif({ assetId: id, ...writes });
- await this.jobRepository.queue({ name: JobName.SidecarWrite, data: { id, ...writes } });
+ await this.assetRepository.upsertExif(
+ updateLockedColumns({
+ assetId: id,
+ ...writes,
+ }),
+ { lockedPropertiesBehavior: 'append' },
+ );
+ await this.jobRepository.queue({ name: JobName.SidecarWrite, data: { id } });
}
}
}
diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts
index c0f930d3a6..98c906d9c7 100644
--- a/server/src/services/metadata.service.spec.ts
+++ b/server/src/services/metadata.service.spec.ts
@@ -187,7 +187,9 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.sidecar.id);
- expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ dateTimeOriginal: sidecarDate }));
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ dateTimeOriginal: sidecarDate }), {
+ lockedPropertiesBehavior: 'skip',
+ });
expect(mocks.asset.update).toHaveBeenCalledWith(
expect.objectContaining({
id: assetStub.image.id,
@@ -214,6 +216,7 @@ describe(MetadataService.name, () => {
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({ dateTimeOriginal: fileModifiedAt }),
+ { lockedPropertiesBehavior: 'skip' },
);
expect(mocks.asset.update).toHaveBeenCalledWith({
id: assetStub.image.id,
@@ -238,7 +241,10 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);
- expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ dateTimeOriginal: fileCreatedAt }));
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+ expect.objectContaining({ dateTimeOriginal: fileCreatedAt }),
+ { lockedPropertiesBehavior: 'skip' },
+ );
expect(mocks.asset.update).toHaveBeenCalledWith({
id: assetStub.image.id,
duration: null,
@@ -258,6 +264,7 @@ describe(MetadataService.name, () => {
expect.objectContaining({
dateTimeOriginal: new Date('2022-01-01T00:00:00.000Z'),
}),
+ { lockedPropertiesBehavior: 'skip' },
);
expect(mocks.asset.update).toHaveBeenCalledWith(
@@ -281,7 +288,9 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);
- expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ iso: 160 }));
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ iso: 160 }), {
+ lockedPropertiesBehavior: 'skip',
+ });
expect(mocks.asset.update).toHaveBeenCalledWith({
id: assetStub.image.id,
duration: null,
@@ -310,6 +319,7 @@ describe(MetadataService.name, () => {
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({ city: null, state: null, country: null }),
+ { lockedPropertiesBehavior: 'skip' },
);
expect(mocks.asset.update).toHaveBeenCalledWith({
id: assetStub.withLocation.id,
@@ -339,6 +349,7 @@ describe(MetadataService.name, () => {
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({ city: 'City', state: 'State', country: 'Country' }),
+ { lockedPropertiesBehavior: 'skip' },
);
expect(mocks.asset.update).toHaveBeenCalledWith({
id: assetStub.withLocation.id,
@@ -358,7 +369,10 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);
- expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ latitude: null, longitude: null }));
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+ expect.objectContaining({ latitude: null, longitude: null }),
+ { lockedPropertiesBehavior: 'skip' },
+ );
});
it('should extract tags from TagsList', async () => {
@@ -571,6 +585,7 @@ describe(MetadataService.name, () => {
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.video.id);
expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
expect.objectContaining({ orientation: ExifOrientation.Rotate270CW.toString() }),
+ { lockedPropertiesBehavior: 'skip' },
);
});
@@ -879,37 +894,40 @@ describe(MetadataService.name, () => {
await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);
- expect(mocks.asset.upsertExif).toHaveBeenCalledWith({
- assetId: assetStub.image.id,
- bitsPerSample: expect.any(Number),
- autoStackId: null,
- colorspace: tags.ColorSpace,
- dateTimeOriginal: dateForTest,
- description: tags.ImageDescription,
- exifImageHeight: null,
- exifImageWidth: null,
- exposureTime: tags.ExposureTime,
- fNumber: null,
- fileSizeInByte: 123_456,
- focalLength: tags.FocalLength,
- fps: null,
- iso: tags.ISO,
- latitude: null,
- lensModel: tags.LensModel,
- livePhotoCID: tags.MediaGroupUUID,
- longitude: null,
- make: tags.Make,
- model: tags.Model,
- modifyDate: expect.any(Date),
- orientation: tags.Orientation?.toString(),
- profileDescription: tags.ProfileDescription,
- projectionType: 'EQUIRECTANGULAR',
- timeZone: tags.tz,
- rating: tags.Rating,
- country: null,
- state: null,
- city: null,
- });
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(
+ {
+ assetId: assetStub.image.id,
+ bitsPerSample: expect.any(Number),
+ autoStackId: null,
+ colorspace: tags.ColorSpace,
+ dateTimeOriginal: dateForTest,
+ description: tags.ImageDescription,
+ exifImageHeight: null,
+ exifImageWidth: null,
+ exposureTime: tags.ExposureTime,
+ fNumber: null,
+ fileSizeInByte: 123_456,
+ focalLength: tags.FocalLength,
+ fps: null,
+ iso: tags.ISO,
+ latitude: null,
+ lensModel: tags.LensModel,
+ livePhotoCID: tags.MediaGroupUUID,
+ longitude: null,
+ make: tags.Make,
+ model: tags.Model,
+ modifyDate: expect.any(Date),
+ orientation: tags.Orientation?.toString(),
+ profileDescription: tags.ProfileDescription,
+ projectionType: 'EQUIRECTANGULAR',
+ timeZone: tags.tz,
+ rating: tags.Rating,
+ country: null,
+ state: null,
+ city: null,
+ },
+ { lockedPropertiesBehavior: 'skip' },
+ );
expect(mocks.asset.update).toHaveBeenCalledWith(
expect.objectContaining({
id: assetStub.image.id,
@@ -943,6 +961,7 @@ describe(MetadataService.name, () => {
expect.objectContaining({
timeZone: 'UTC+0',
}),
+ { lockedPropertiesBehavior: 'skip' },
);
});
@@ -1103,6 +1122,7 @@ describe(MetadataService.name, () => {
expect.objectContaining({
description: '',
}),
+ { lockedPropertiesBehavior: 'skip' },
);
mockReadTags({ ImageDescription: ' my\n description' });
@@ -1111,6 +1131,7 @@ describe(MetadataService.name, () => {
expect.objectContaining({
description: 'my\n description',
}),
+ { lockedPropertiesBehavior: 'skip' },
);
});
@@ -1123,6 +1144,7 @@ describe(MetadataService.name, () => {
expect.objectContaining({
description: '1000',
}),
+ { lockedPropertiesBehavior: 'skip' },
);
});
@@ -1346,6 +1368,7 @@ describe(MetadataService.name, () => {
expect.objectContaining({
modifyDate: expect.any(Date),
}),
+ { lockedPropertiesBehavior: 'skip' },
);
});
@@ -1358,6 +1381,7 @@ describe(MetadataService.name, () => {
expect.objectContaining({
rating: null,
}),
+ { lockedPropertiesBehavior: 'skip' },
);
});
@@ -1370,6 +1394,7 @@ describe(MetadataService.name, () => {
expect.objectContaining({
rating: 5,
}),
+ { lockedPropertiesBehavior: 'skip' },
);
});
@@ -1382,6 +1407,7 @@ describe(MetadataService.name, () => {
expect.objectContaining({
rating: -1,
}),
+ { lockedPropertiesBehavior: 'skip' },
);
});
@@ -1503,7 +1529,9 @@ describe(MetadataService.name, () => {
mockReadTags(exif);
await sut.handleMetadataExtraction({ id: assetStub.image.id });
- expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining(expected));
+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining(expected), {
+ lockedPropertiesBehavior: 'skip',
+ });
});
it.each([
@@ -1529,6 +1557,7 @@ describe(MetadataService.name, () => {
expect.objectContaining({
lensModel: expected,
}),
+ { lockedPropertiesBehavior: 'skip' },
);
});
});
@@ -1637,12 +1666,14 @@ describe(MetadataService.name, () => {
describe('handleSidecarWrite', () => {
it('should skip assets that no longer exist', async () => {
+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);
mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(void 0);
await expect(sut.handleSidecarWrite({ id: 'asset-123' })).resolves.toBe(JobStatus.Failed);
expect(mocks.metadata.writeTags).not.toHaveBeenCalled();
});
it('should skip jobs with no metadata', async () => {
+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);
const asset = factory.jobAssets.sidecarWrite();
mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(asset);
await expect(sut.handleSidecarWrite({ id: asset.id })).resolves.toBe(JobStatus.Skipped);
@@ -1655,20 +1686,22 @@ describe(MetadataService.name, () => {
const gps = 12;
const date = '2023-11-22T04:56:12.196Z';
+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([
+ 'description',
+ 'latitude',
+ 'longitude',
+ 'dateTimeOriginal',
+ ]);
mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(asset);
await expect(
sut.handleSidecarWrite({
id: asset.id,
- description,
- latitude: gps,
- longitude: gps,
- dateTimeOriginal: date,
}),
).resolves.toBe(JobStatus.Success);
expect(mocks.metadata.writeTags).toHaveBeenCalledWith(asset.files[0].path, {
+ DateTimeOriginal: date,
Description: description,
ImageDescription: description,
- DateTimeOriginal: date,
GPSLatitude: gps,
GPSLongitude: gps,
});
diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts
index a1267604b2..8da7c88ecc 100644
--- a/server/src/services/metadata.service.ts
+++ b/server/src/services/metadata.service.ts
@@ -290,7 +290,7 @@ export class MetadataService extends BaseService {
};
const promises: Promise<unknown>[] = [
- this.assetRepository.upsertExif(exifData),
+ this.assetRepository.upsertExif(exifData, { lockedPropertiesBehavior: 'skip' }),
this.assetRepository.update({
id: asset.id,
duration: this.getDuration(exifTags),
@@ -393,22 +393,34 @@ export class MetadataService extends BaseService {
@OnJob({ name: JobName.SidecarWrite, queue: QueueName.Sidecar })
async handleSidecarWrite(job: JobOf<JobName.SidecarWrite>): Promise<JobStatus> {
- const { id, description, dateTimeOriginal, latitude, longitude, rating, tags } = job;
+ const { id, tags } = job;
const asset = await this.assetJobRepository.getForSidecarWriteJob(id);
if (!asset) {
return JobStatus.Failed;
}
+ const lockedProperties = await this.assetJobRepository.getLockedPropertiesForMetadataExtraction(id);
const tagsList = (asset.tags || []).map((tag) => tag.value);
const { sidecarFile } = getAssetFiles(asset.files);
const sidecarPath = sidecarFile?.path || `${asset.originalPath}.xmp`;
+ const { description, dateTimeOriginal, latitude, longitude, rating } = _.pick(
+ {
+ description: asset.exifInfo.description,
+ dateTimeOriginal: asset.exifInfo.dateTimeOriginal,
+ latitude: asset.exifInfo.latitude,
+ longitude: asset.exifInfo.longitude,
+ rating: asset.exifInfo.rating,
+ },
+ lockedProperties,
+ );
+
const exif = _.omitBy(
<Tags>{
Description: description,
ImageDescription: description,
- DateTimeOriginal: dateTimeOriginal,
+ DateTimeOriginal: dateTimeOriginal ? String(dateTimeOriginal) : undefined,
GPSLatitude: latitude,
GPSLongitude: longitude,
Rating: rating,
diff --git a/server/src/types.ts b/server/src/types.ts
index a33dba490c..e404332fac 100644
--- a/server/src/types.ts
+++ b/server/src/types.ts
@@ -222,11 +222,6 @@ export interface IDeleteFilesJob extends IBaseJob {
}
export interface ISidecarWriteJob extends IEntityJob {
- description?: string;
- dateTimeOriginal?: string;
- latitude?: number;
- longitude?: number;
- rating?: number;
tags?: true;
}
diff --git a/server/src/utils/database.ts b/server/src/utils/database.ts
index f8dbd5e78c..656e8e628a 100644
--- a/server/src/utils/database.ts
+++ b/server/src/utils/database.ts
@@ -15,7 +15,7 @@ import { PostgresJSDialect } from 'kysely-postgres-js';
import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
import { parse } from 'pg-connection-string';
import postgres, { Notice, PostgresError } from 'postgres';
-import { columns, Exif, Person } from 'src/database';
+import { columns, Exif, lockableProperties, LockableProperty, Person } from 'src/database';
import { AssetFileType, AssetVisibility, DatabaseExtension, DatabaseSslMode } from 'src/enum';
import { AssetSearchBuilderOptions } from 'src/repositories/search.repository';
import { DB } from 'src/schema';
@@ -488,3 +488,10 @@ export function vectorIndexQuery({ vectorExtension, table, indexName, lists }: V
}
}
}
+
+export const updateLockedColumns = <T extends Record<string, unknown> & { lockedProperties?: LockableProperty[] }>(
+ exif: T,
+) => {
+ exif.lockedProperties = lockableProperties.filter((property) => property in exif);
+ return exif;
+};
diff --git a/server/test/medium.factory.ts b/server/test/medium.factory.ts
index efcdc59793..82ea2cd1fc 100644
--- a/server/test/medium.factory.ts
+++ b/server/test/medium.factory.ts
@@ -202,7 +202,7 @@ export class MediumTestContext<S extends BaseService = BaseService> {
}
async newExif(dto: Insertable<AssetExifTable>) {
- const result = await this.get(AssetRepository).upsertExif(dto);
+ const result = await this.get(AssetRepository).upsertExif(dto, { lockedPropertiesBehavior: 'override' });
return { result };
}
diff --git a/server/test/medium/specs/services/asset.service.spec.ts b/server/test/medium/specs/services/asset.service.spec.ts
index 13bc1ca9a9..8b54019fcf 100644
--- a/server/test/medium/specs/services/asset.service.spec.ts
+++ b/server/test/medium/specs/services/asset.service.spec.ts
@@ -268,4 +268,65 @@ describe(AssetService.name, () => {
});
});
});
+
+ describe('update', () => {
+ it('should update dateTimeOriginal', async () => {
+ const { sut, ctx } = setup();
+ ctx.getMock(JobRepository).queue.mockResolvedValue();
+ const { user } = await ctx.newUser();
+ const auth = factory.auth({ user });
+ const { asset } = await ctx.newAsset({ ownerId: user.id });
+ await ctx.newExif({ assetId: asset.id, description: 'test' });
+
+ await expect(
+ ctx.database
+ .selectFrom('asset_exif')
+ .select('lockedProperties')
+ .where('assetId', '=', asset.id)
+ .executeTakeFirstOrThrow(),
+ ).resolves.toEqual({ lockedProperties: null });
+ await sut.update(auth, asset.id, { dateTimeOriginal: '2023-11-19T18:11:00.000-07:00' });
+
+ await expect(
+ ctx.database
+ .selectFrom('asset_exif')
+ .select('lockedProperties')
+ .where('assetId', '=', asset.id)
+ .executeTakeFirstOrThrow(),
+ ).resolves.toEqual({ lockedProperties: ['dateTimeOriginal'] });
+ await expect(ctx.get(AssetRepository).getById(asset.id, { exifInfo: true })).resolves.toEqual(
+ expect.objectContaining({
+ exifInfo: expect.objectContaining({ dateTimeOriginal: '2023-11-20T01:11:00+00:00' }),
+ }),
+ );
+ });
+ });
+
+ describe('updateAll', () => {
+ it('should relatively update assets', async () => {
+ const { sut, ctx } = setup();
+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();
+ const { user } = await ctx.newUser();
+ const auth = factory.auth({ user });
+ const { asset } = await ctx.newAsset({ ownerId: user.id });
+ await ctx.newExif({ assetId: asset.id, dateTimeOriginal: '2023-11-19T18:11:00' });
+
+ await sut.updateAll(auth, { ids: [asset.id], dateTimeRelative: -11 });
+
+ await expect(
+ ctx.database
+ .selectFrom('asset_exif')
+ .select('lockedProperties')
+ .where('assetId', '=', asset.id)
+ .executeTakeFirstOrThrow(),
+ ).resolves.toEqual({ lockedProperties: ['timeZone', 'dateTimeOriginal'] });
+ await expect(ctx.get(AssetRepository).getById(asset.id, { exifInfo: true })).resolves.toEqual(
+ expect.objectContaining({
+ exifInfo: expect.objectContaining({
+ dateTimeOriginal: '2023-11-19T18:00:00+00:00',
+ }),
+ }),
+ );
+ });
+ });
});
diff --git a/server/test/medium/specs/services/metadata.service.spec.ts b/server/test/medium/specs/services/metadata.service.spec.ts
index 1d144e9c9c..4c3499857d 100644
--- a/server/test/medium/specs/services/metadata.service.spec.ts
+++ b/server/test/medium/specs/services/metadata.service.spec.ts
@@ -95,6 +95,7 @@ describe(MetadataService.name, () => {
dateTimeOriginal: new Date(expected.dateTimeOriginal),
timeZone: expected.timeZone,
}),
+ { lockedPropertiesBehavior: 'skip' },
);
expect(mocks.asset.update).toHaveBeenCalledWith(
diff --git a/server/test/medium/specs/sync/sync-album-asset-exif.spec.ts b/server/test/medium/specs/sync/sync-album-asset-exif.spec.ts
index fd563f4db1..1865fc2c80 100644
--- a/server/test/medium/specs/sync/sync-album-asset-exif.spec.ts
+++ b/server/test/medium/specs/sync/sync-album-asset-exif.spec.ts
@@ -2,6 +2,7 @@ import { Kysely } from 'kysely';
import { AlbumUserRole, SyncEntityType, SyncRequestType } from 'src/enum';
import { AssetRepository } from 'src/repositories/asset.repository';
import { DB } from 'src/schema';
+import { updateLockedColumns } from 'src/utils/database';
import { SyncTestContext } from 'test/medium.factory';
import { factory } from 'test/small.factory';
import { getKyselyDB, wait } from 'test/utils';
@@ -288,10 +289,13 @@ describe(SyncRequestType.AlbumAssetExifsV1, () => {
// update the asset
const assetRepository = ctx.get(AssetRepository);
- await assetRepository.upsertExif({
- assetId: asset.id,
- city: 'New City',
- });
+ await assetRepository.upsertExif(
+ updateLockedColumns({
+ assetId: asset.id,
+ city: 'New City',
+ }),
+ { lockedPropertiesBehavior: 'append' },
+ );
await expect(ctx.syncStream(auth, [SyncRequestType.AlbumAssetExifsV1])).resolves.toEqual([
{
@@ -346,10 +350,13 @@ describe(SyncRequestType.AlbumAssetExifsV1, () => {
// update the asset
const assetRepository = ctx.get(AssetRepository);
- await assetRepository.upsertExif({
- assetId: assetDelayedExif.id,
- city: 'Delayed Exif',
- });
+ await assetRepository.upsertExif(
+ updateLockedColumns({
+ assetId: assetDelayedExif.id,
+ city: 'Delayed Exif',
+ }),
+ { lockedPropertiesBehavior: 'append' },
+ );
await expect(ctx.syncStream(auth, [SyncRequestType.AlbumAssetExifsV1])).resolves.toEqual([
{
diff --git a/server/test/small.factory.ts b/server/test/small.factory.ts
index 13cccce176..53d7c70794 100644
--- a/server/test/small.factory.ts
+++ b/server/test/small.factory.ts
@@ -4,6 +4,7 @@ import {
AuthApiKey,
AuthSharedLink,
AuthUser,
+ Exif,
Library,
Memory,
Partner,
@@ -319,18 +320,28 @@ const versionHistoryFactory = () => ({
version: '1.123.45',
});
-const assetSidecarWriteFactory = () => ({
- id: newUuid(),
- originalPath: '/path/to/original-path.jpg.xmp',
- tags: [],
- files: [
- {
- id: newUuid(),
- path: '/path/to/original-path.jpg.xmp',
- type: AssetFileType.Sidecar,
- },
- ],
-});
+const assetSidecarWriteFactory = () => {
+ const id = newUuid();
+ return {
+ id,
+ originalPath: '/path/to/original-path.jpg.xmp',
+ tags: [],
+ files: [
+ {
+ id: newUuid(),
+ path: '/path/to/original-path.jpg.xmp',
+ type: AssetFileType.Sidecar,
+ },
+ ],
+ exifInfo: {
+ assetId: id,
+ description: 'this is a description',
+ latitude: 12,
+ longitude: 12,
+ dateTimeOriginal: '2023-11-22T04:56:12.196Z',
+ } as unknown as Exif,
+ };
+};
const assetOcrFactory = (
ocr: {