High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
fix: asset update race condition (#24384)
Daniel Dietzlerc15998e80595
diff --git a/server/src/database.ts b/server/src/database.tsindex a3c38ae61e..9f4494b720 100644--- a/server/src/database.ts+++ b/server/src/database.ts@@ -240,7 +240,7 @@ export type Session = {isPendingSyncReset: boolean;};-export type Exif = Omit<Selectable<AssetExifTable>, 'updatedAt' | 'updateId'>;+export type Exif = Omit<Selectable<AssetExifTable>, 'updatedAt' | 'updateId' | 'lockedProperties'>;export type Person = {createdAt: Date;@@ -465,3 +465,13 @@ export const columns = {'plugin.updatedAt as updatedAt',],} as const;++export type LockableProperty = (typeof lockableProperties)[number];+export const lockableProperties = [+ 'description',+ 'dateTimeOriginal',+ 'latitude',+ 'longitude',+ 'rating',+ 'timeZone',+] as const;diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sqlindex d1b4e5a72e..b736998e28 100644--- a/server/src/queries/asset.job.repository.sql+++ b/server/src/queries/asset.job.repository.sql@@ -50,9 +50,11 @@ selectwhere"asset"."id" = "tag_asset"."assetId") as agg- ) as "tags"+ ) as "tags",+ to_json("asset_exif") as "exifInfo"from"asset"+ inner join "asset_exif" on "asset"."id" = "asset_exif"."assetId"where"asset"."id" = $2::uuidlimit@@ -224,6 +226,14 @@ fromwhere"asset"."id" = $2+-- AssetJobRepository.getLockedPropertiesForMetadataExtraction+select+ "asset_exif"."lockedProperties"+from+ "asset_exif"+where+ "asset_exif"."assetId" = $1+-- AssetJobRepository.getAlbumThumbnailFilesselect"asset_file"."id",diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sqlindex 01cc6a7a89..f25a0798d2 100644--- a/server/src/queries/asset.repository.sql+++ b/server/src/queries/asset.repository.sql@@ -1,19 +1,49 @@-- NOTE: This file is auto generated by ./sql-generator+-- AssetRepository.upsertExif+insert into+ "asset_exif" ("dateTimeOriginal", "lockedProperties")+values+ ($1, $2)+on conflict ("assetId") do update+set+ "dateTimeOriginal" = "excluded"."dateTimeOriginal",+ "lockedProperties" = nullif(+ array(+ select distinct+ unnest("asset_exif"."lockedProperties" || $3)+ ),+ '{}'+ )+-- AssetRepository.updateAllExifupdate "asset_exif"set- "model" = $1+ "model" = $1,+ "lockedProperties" = nullif(+ array(+ select distinct+ unnest("asset_exif"."lockedProperties" || $2)+ ),+ '{}'+ )where- "assetId" in ($2)+ "assetId" in ($3)-- AssetRepository.updateDateTimeOriginalupdate "asset_exif"set"dateTimeOriginal" = "dateTimeOriginal" + $1::interval,- "timeZone" = $2+ "timeZone" = $2,+ "lockedProperties" = nullif(+ array(+ select distinct+ unnest("asset_exif"."lockedProperties" || $3)+ ),+ '{}'+ )where- "assetId" in ($3)+ "assetId" in ($4)returning"assetId","dateTimeOriginal",diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.tsindex de994b08cd..214d42747f 100644--- a/server/src/repositories/asset-job.repository.ts+++ b/server/src/repositories/asset-job.repository.ts@@ -50,6 +50,7 @@ export class AssetJobRepository {.whereRef('asset.id', '=', 'tag_asset.assetId'),).as('tags'),)+ .$call(withExifInner).limit(1).executeTakeFirst();}@@ -128,6 +129,16 @@ export class AssetJobRepository {.executeTakeFirst();}+ @GenerateSql({ params: [DummyValue.UUID] })+ async getLockedPropertiesForMetadataExtraction(assetId: string) {+ return this.db+ .selectFrom('asset_exif')+ .select('asset_exif.lockedProperties')+ .where('asset_exif.assetId', '=', assetId)+ .executeTakeFirst()+ .then((row) => row?.lockedProperties ?? []);+ }+@GenerateSql({ params: [DummyValue.UUID, AssetFileType.Thumbnail] })getAlbumThumbnailFiles(id: string, fileType?: AssetFileType) {return this.dbdiff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.tsindex 4b8cbd7a7a..7db3a76f12 100644--- a/server/src/repositories/asset.repository.ts+++ b/server/src/repositories/asset.repository.ts@@ -1,8 +1,8 @@import { Injectable } from '@nestjs/common';-import { Insertable, Kysely, NotNull, Selectable, sql, Updateable, UpdateResult } from 'kysely';+import { ExpressionBuilder, Insertable, Kysely, NotNull, Selectable, sql, Updateable, UpdateResult } from 'kysely';import { isEmpty, isUndefined, omitBy } from 'lodash';import { InjectKysely } from 'nestjs-kysely';-import { Stack } from 'src/database';+import { LockableProperty, Stack } from 'src/database';import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';import { AuthDto } from 'src/dtos/auth.dto';import { AssetFileType, AssetMetadataKey, AssetOrder, AssetStatus, AssetType, AssetVisibility } from 'src/enum';@@ -113,51 +113,77 @@ interface GetByIdsRelations {tags?: boolean;}+const distinctLocked = <T extends LockableProperty[] | null>(eb: ExpressionBuilder<DB, 'asset_exif'>, columns: T) =>+ sql<T>`nullif(array(select distinct unnest(${eb.ref('asset_exif.lockedProperties')} || ${columns})), '{}')`;+@Injectable()export class AssetRepository {constructor(@InjectKysely() private db: Kysely<DB>) {}- async upsertExif(exif: Insertable<AssetExifTable>): Promise<void> {- const value = { ...exif, assetId: asUuid(exif.assetId) };+ @GenerateSql({+ params: [+ { dateTimeOriginal: DummyValue.DATE, lockedProperties: ['dateTimeOriginal'] },+ { lockedPropertiesBehavior: 'append' },+ ],+ })+ async upsertExif(+ exif: Insertable<AssetExifTable>,+ { lockedPropertiesBehavior }: { lockedPropertiesBehavior: 'override' | 'append' | 'skip' },+ ): Promise<void> {await this.db.insertInto('asset_exif')- .values(value)+ .values(exif).onConflict((oc) =>- oc.column('assetId').doUpdateSet((eb) =>- removeUndefinedKeys(- {- description: eb.ref('excluded.description'),- exifImageWidth: eb.ref('excluded.exifImageWidth'),- exifImageHeight: eb.ref('excluded.exifImageHeight'),- fileSizeInByte: eb.ref('excluded.fileSizeInByte'),- orientation: eb.ref('excluded.orientation'),- dateTimeOriginal: eb.ref('excluded.dateTimeOriginal'),- modifyDate: eb.ref('excluded.modifyDate'),- timeZone: eb.ref('excluded.timeZone'),- latitude: eb.ref('excluded.latitude'),- longitude: eb.ref('excluded.longitude'),- projectionType: eb.ref('excluded.projectionType'),- city: eb.ref('excluded.city'),- livePhotoCID: eb.ref('excluded.livePhotoCID'),- autoStackId: eb.ref('excluded.autoStackId'),- state: eb.ref('excluded.state'),- country: eb.ref('excluded.country'),- make: eb.ref('excluded.make'),- model: eb.ref('excluded.model'),- lensModel: eb.ref('excluded.lensModel'),- fNumber: eb.ref('excluded.fNumber'),- focalLength: eb.ref('excluded.focalLength'),- iso: eb.ref('excluded.iso'),- exposureTime: eb.ref('excluded.exposureTime'),- profileDescription: eb.ref('excluded.profileDescription'),- colorspace: eb.ref('excluded.colorspace'),- bitsPerSample: eb.ref('excluded.bitsPerSample'),- rating: eb.ref('excluded.rating'),- fps: eb.ref('excluded.fps'),- },- value,- ),- ),+ oc.column('assetId').doUpdateSet((eb) => {+ const updateLocked = <T extends keyof AssetExifTable>(col: T) => eb.ref(`excluded.${col}`);+ const skipLocked = <T extends keyof AssetExifTable>(col: T) =>+ eb+ .case()+ .when(sql`${col}`, '=', eb.fn.any('asset_exif.lockedProperties'))+ .then(eb.ref(`asset_exif.${col}`))+ .else(eb.ref(`excluded.${col}`))+ .end();+ const ref = lockedPropertiesBehavior === 'skip' ? skipLocked : updateLocked;+ return {+ ...removeUndefinedKeys(+ {+ description: ref('description'),+ exifImageWidth: ref('exifImageWidth'),+ exifImageHeight: ref('exifImageHeight'),+ fileSizeInByte: ref('fileSizeInByte'),+ orientation: ref('orientation'),+ dateTimeOriginal: ref('dateTimeOriginal'),+ modifyDate: ref('modifyDate'),+ timeZone: ref('timeZone'),+ latitude: ref('latitude'),+ longitude: ref('longitude'),+ projectionType: ref('projectionType'),+ city: ref('city'),+ livePhotoCID: ref('livePhotoCID'),+ autoStackId: ref('autoStackId'),+ state: ref('state'),+ country: ref('country'),+ make: ref('make'),+ model: ref('model'),+ lensModel: ref('lensModel'),+ fNumber: ref('fNumber'),+ focalLength: ref('focalLength'),+ iso: ref('iso'),+ exposureTime: ref('exposureTime'),+ profileDescription: ref('profileDescription'),+ colorspace: ref('colorspace'),+ bitsPerSample: ref('bitsPerSample'),+ rating: ref('rating'),+ fps: ref('fps'),+ lockedProperties:+ lockedPropertiesBehavior === 'append'+ ? distinctLocked(eb, exif.lockedProperties ?? null)+ : ref('lockedProperties'),+ },+ exif,+ ),+ };+ }),).execute();}@@ -169,19 +195,26 @@ export class AssetRepository {return;}- await this.db.updateTable('asset_exif').set(options).where('assetId', 'in', ids).execute();+ await this.db+ .updateTable('asset_exif')+ .set((eb) => ({+ ...options,+ lockedProperties: distinctLocked(eb, Object.keys(options) as LockableProperty[]),+ }))+ .where('assetId', 'in', ids)+ .execute();}@GenerateSql({ params: [[DummyValue.UUID], DummyValue.NUMBER, DummyValue.STRING] })@Chunked()- async updateDateTimeOriginal(- ids: string[],- delta?: number,- timeZone?: string,- ): Promise<{ assetId: string; dateTimeOriginal: Date | null; timeZone: string | null }[]> {- return await this.db+ updateDateTimeOriginal(ids: string[], delta?: number, timeZone?: string) {+ return this.db.updateTable('asset_exif')- .set({ dateTimeOriginal: sql`"dateTimeOriginal" + ${(delta ?? 0) + ' minute'}::interval`, timeZone })+ .set((eb) => ({+ dateTimeOriginal: sql`"dateTimeOriginal" + ${(delta ?? 0) + ' minute'}::interval`,+ timeZone,+ lockedProperties: distinctLocked(eb, ['dateTimeOriginal', 'timeZone']),+ })).where('assetId', 'in', ids).returning(['assetId', 'dateTimeOriginal', 'timeZone']).execute();diff --git a/server/src/schema/migrations/1764957138636-AddLockedPropertiesToAssetExif.ts b/server/src/schema/migrations/1764957138636-AddLockedPropertiesToAssetExif.tsnew file mode 100644index 0000000000..0cd024a4d8--- /dev/null+++ b/server/src/schema/migrations/1764957138636-AddLockedPropertiesToAssetExif.ts@@ -0,0 +1,9 @@+import { Kysely, sql } from 'kysely';++export async function up(db: Kysely<any>): Promise<void> {+ await sql`ALTER TABLE "asset_exif" ADD "lockedProperties" character varying[];`.execute(db);+}++export async function down(db: Kysely<any>): Promise<void> {+ await sql`ALTER TABLE "asset_exif" DROP COLUMN "lockedProperties";`.execute(db);+}diff --git a/server/src/schema/tables/asset-exif.table.ts b/server/src/schema/tables/asset-exif.table.tsindex f28735a2db..346098a72c 100644--- a/server/src/schema/tables/asset-exif.table.ts+++ b/server/src/schema/tables/asset-exif.table.ts@@ -1,3 +1,4 @@+import { LockableProperty } from 'src/database';import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators';import { AssetTable } from 'src/schema/tables/asset.table';import { Column, ForeignKeyColumn, Generated, Int8, Table, Timestamp, UpdateDateColumn } from 'src/sql-tools';@@ -97,4 +98,7 @@ export class AssetExifTable {@UpdateIdColumn({ index: true })updateId!: Generated<string>;++ @Column({ type: 'character varying', array: true, nullable: true })+ lockedProperties!: Array<LockableProperty> | null;}diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.tsindex d2e1c14210..2bb8530c1c 100644--- a/server/src/services/asset-media.service.ts+++ b/server/src/services/asset-media.service.ts@@ -370,7 +370,10 @@ export class AssetMediaService extends BaseService {: this.assetRepository.deleteFile({ assetId, type: AssetFileType.Sidecar }));await this.storageRepository.utimes(file.originalPath, new Date(), new Date(dto.fileModifiedAt));- await this.assetRepository.upsertExif({ assetId, fileSizeInByte: file.size });+ await this.assetRepository.upsertExif(+ { assetId, fileSizeInByte: file.size },+ { lockedPropertiesBehavior: 'override' },+ );await this.jobRepository.queue({name: JobName.AssetExtractMetadata,data: { id: assetId, source: 'upload' },@@ -399,7 +402,10 @@ export class AssetMediaService extends BaseService {});const { size } = await this.storageRepository.stat(created.originalPath);- await this.assetRepository.upsertExif({ assetId: created.id, fileSizeInByte: size });+ await this.assetRepository.upsertExif(+ { assetId: created.id, fileSizeInByte: size },+ { lockedPropertiesBehavior: 'override' },+ );await this.jobRepository.queue({ name: JobName.AssetExtractMetadata, data: { id: created.id, source: 'copy' } });return created;}@@ -440,7 +446,10 @@ export class AssetMediaService extends BaseService {await this.storageRepository.utimes(sidecarFile.originalPath, new Date(), new Date(dto.fileModifiedAt));}await this.storageRepository.utimes(file.originalPath, new Date(), new Date(dto.fileModifiedAt));- await this.assetRepository.upsertExif({ assetId: asset.id, fileSizeInByte: file.size });+ await this.assetRepository.upsertExif(+ { assetId: asset.id, fileSizeInByte: file.size },+ { lockedPropertiesBehavior: 'override' },+ );await this.eventRepository.emit('AssetCreate', { asset });diff --git a/server/src/services/asset.service.spec.ts b/server/src/services/asset.service.spec.tsindex 878721e0a7..5e1cce2ccf 100755--- a/server/src/services/asset.service.spec.ts+++ b/server/src/services/asset.service.spec.ts@@ -225,7 +225,10 @@ describe(AssetService.name, () => {await sut.update(authStub.admin, 'asset-1', { description: 'Test description' });- expect(mocks.asset.upsertExif).toHaveBeenCalledWith({ assetId: 'asset-1', description: 'Test description' });+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(+ { assetId: 'asset-1', description: 'Test description', lockedProperties: ['description'] },+ { lockedPropertiesBehavior: 'append' },+ );});it('should update the exif rating', async () => {@@ -235,7 +238,14 @@ describe(AssetService.name, () => {await sut.update(authStub.admin, 'asset-1', { rating: 3 });- expect(mocks.asset.upsertExif).toHaveBeenCalledWith({ assetId: 'asset-1', rating: 3 });+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(+ {+ assetId: 'asset-1',+ rating: 3,+ lockedProperties: ['rating'],+ },+ { lockedPropertiesBehavior: 'append' },+ );});it('should fail linking a live video if the motion part could not be found', async () => {@@ -427,9 +437,7 @@ describe(AssetService.name, () => {});expect(mocks.asset.updateAll).toHaveBeenCalled();expect(mocks.asset.updateAllExif).toHaveBeenCalledWith(['asset-1'], { latitude: 0, longitude: 0 });- expect(mocks.job.queueAll).toHaveBeenCalledWith([- { name: JobName.SidecarWrite, data: { id: 'asset-1', latitude: 0, longitude: 0 } },- ]);+ expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.SidecarWrite, data: { id: 'asset-1' } }]);});it('should update exif table if latitude field is provided', async () => {@@ -450,9 +458,7 @@ describe(AssetService.name, () => {latitude: 30,longitude: 50,});- expect(mocks.job.queueAll).toHaveBeenCalledWith([- { name: JobName.SidecarWrite, data: { id: 'asset-1', dateTimeOriginal, latitude: 30, longitude: 50 } },- ]);+ expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.SidecarWrite, data: { id: 'asset-1' } }]);});it('should update Assets table if duplicateId is provided as null', async () => {@@ -482,18 +488,7 @@ describe(AssetService.name, () => {timeZone,});expect(mocks.asset.updateDateTimeOriginal).toHaveBeenCalledWith(['asset-1'], dateTimeRelative, timeZone);- expect(mocks.job.queueAll).toHaveBeenCalledWith([- {- name: JobName.SidecarWrite,- data: {- id: 'asset-1',- dateTimeOriginal: '2020-02-25T06:41:00.000+02:00',- description: undefined,- latitude: undefined,- longitude: undefined,- },- },- ]);+ expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.SidecarWrite, data: { id: 'asset-1' } }]);});});diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.tsindex 32c6526394..282d74a9b1 100644--- a/server/src/services/asset.service.ts+++ b/server/src/services/asset.service.ts@@ -30,9 +30,10 @@ import {QueueName,} from 'src/enum';import { BaseService } from 'src/services/base.service';-import { ISidecarWriteJob, JobItem, JobOf } from 'src/types';+import { JobItem, JobOf } from 'src/types';import { requireElevatedPermission } from 'src/utils/access';import { getAssetFiles, getMyPartnerIds, onAfterUnlink, onBeforeLink, onBeforeUnlink } from 'src/utils/asset.util';+import { updateLockedColumns } from 'src/utils/database';@Injectable()export class AssetService extends BaseService {@@ -142,56 +143,26 @@ export class AssetService extends BaseService {} = dto;await this.requireAccess({ auth, permission: Permission.AssetUpdate, ids });- const assetDto = { isFavorite, visibility, duplicateId };- const exifDto = { latitude, longitude, rating, description, dateTimeOriginal };+ const assetDto = _.omitBy({ isFavorite, visibility, duplicateId }, _.isUndefined);+ const exifDto = _.omitBy({ latitude, longitude, rating, description, dateTimeOriginal }, _.isUndefined);- const isExifChanged = Object.values(exifDto).some((v) => v !== undefined);- if (isExifChanged) {+ if (Object.keys(exifDto).length > 0) {await this.assetRepository.updateAllExif(ids, exifDto);}- const assets =- (dateTimeRelative !== undefined && dateTimeRelative !== 0) || timeZone !== undefined- ? await this.assetRepository.updateDateTimeOriginal(ids, dateTimeRelative, timeZone)- : undefined;-- const dateTimesWithTimezone = assets- ? assets.map((asset) => {- const isoString = asset.dateTimeOriginal?.toISOString();- let dateTime = isoString ? DateTime.fromISO(isoString) : null;-- if (dateTime && asset.timeZone) {- dateTime = dateTime.setZone(asset.timeZone);- }-- return {- assetId: asset.assetId,- dateTimeOriginal: dateTime?.toISO() ?? null,- };- })- : ids.map((id) => ({ assetId: id, dateTimeOriginal }));-- if (dateTimesWithTimezone.length > 0) {- await this.jobRepository.queueAll(- dateTimesWithTimezone.map(({ assetId: id, dateTimeOriginal }) => ({- name: JobName.SidecarWrite,- data: {- ...exifDto,- id,- dateTimeOriginal: dateTimeOriginal ?? undefined,- },- })),- );+ if ((dateTimeRelative !== undefined && dateTimeRelative !== 0) || timeZone !== undefined) {+ await this.assetRepository.updateDateTimeOriginal(ids, dateTimeRelative, timeZone);}- const isAssetChanged = Object.values(assetDto).some((v) => v !== undefined);- if (isAssetChanged) {+ if (Object.keys(assetDto).length > 0) {await this.assetRepository.updateAll(ids, assetDto);+ }- if (visibility === AssetVisibility.Locked) {- await this.albumRepository.removeAssetsFromAll(ids);- }+ if (visibility === AssetVisibility.Locked) {+ await this.albumRepository.removeAssetsFromAll(ids);}++ await this.jobRepository.queueAll(ids.map((id) => ({ name: JobName.SidecarWrite, data: { id } })));}async copy(@@ -456,12 +427,25 @@ export class AssetService extends BaseService {return asset;}- private async updateExif(dto: ISidecarWriteJob) {+ private async updateExif(dto: {+ id: string;+ description?: string;+ dateTimeOriginal?: string;+ latitude?: number;+ longitude?: number;+ rating?: number;+ }) {const { id, description, dateTimeOriginal, latitude, longitude, rating } = dto;const writes = _.omitBy({ description, dateTimeOriginal, latitude, longitude, rating }, _.isUndefined);if (Object.keys(writes).length > 0) {- await this.assetRepository.upsertExif({ assetId: id, ...writes });- await this.jobRepository.queue({ name: JobName.SidecarWrite, data: { id, ...writes } });+ await this.assetRepository.upsertExif(+ updateLockedColumns({+ assetId: id,+ ...writes,+ }),+ { lockedPropertiesBehavior: 'append' },+ );+ await this.jobRepository.queue({ name: JobName.SidecarWrite, data: { id } });}}}diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.tsindex c0f930d3a6..98c906d9c7 100644--- a/server/src/services/metadata.service.spec.ts+++ b/server/src/services/metadata.service.spec.ts@@ -187,7 +187,9 @@ describe(MetadataService.name, () => {await sut.handleMetadataExtraction({ id: assetStub.image.id });expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.sidecar.id);- expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ dateTimeOriginal: sidecarDate }));+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ dateTimeOriginal: sidecarDate }), {+ lockedPropertiesBehavior: 'skip',+ });expect(mocks.asset.update).toHaveBeenCalledWith(expect.objectContaining({id: assetStub.image.id,@@ -214,6 +216,7 @@ describe(MetadataService.name, () => {expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ dateTimeOriginal: fileModifiedAt }),+ { lockedPropertiesBehavior: 'skip' },);expect(mocks.asset.update).toHaveBeenCalledWith({id: assetStub.image.id,@@ -238,7 +241,10 @@ describe(MetadataService.name, () => {await sut.handleMetadataExtraction({ id: assetStub.image.id });expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);- expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ dateTimeOriginal: fileCreatedAt }));+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(+ expect.objectContaining({ dateTimeOriginal: fileCreatedAt }),+ { lockedPropertiesBehavior: 'skip' },+ );expect(mocks.asset.update).toHaveBeenCalledWith({id: assetStub.image.id,duration: null,@@ -258,6 +264,7 @@ describe(MetadataService.name, () => {expect.objectContaining({dateTimeOriginal: new Date('2022-01-01T00:00:00.000Z'),}),+ { lockedPropertiesBehavior: 'skip' },);expect(mocks.asset.update).toHaveBeenCalledWith(@@ -281,7 +288,9 @@ describe(MetadataService.name, () => {await sut.handleMetadataExtraction({ id: assetStub.image.id });expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);- expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ iso: 160 }));+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ iso: 160 }), {+ lockedPropertiesBehavior: 'skip',+ });expect(mocks.asset.update).toHaveBeenCalledWith({id: assetStub.image.id,duration: null,@@ -310,6 +319,7 @@ describe(MetadataService.name, () => {expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ city: null, state: null, country: null }),+ { lockedPropertiesBehavior: 'skip' },);expect(mocks.asset.update).toHaveBeenCalledWith({id: assetStub.withLocation.id,@@ -339,6 +349,7 @@ describe(MetadataService.name, () => {expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ city: 'City', state: 'State', country: 'Country' }),+ { lockedPropertiesBehavior: 'skip' },);expect(mocks.asset.update).toHaveBeenCalledWith({id: assetStub.withLocation.id,@@ -358,7 +369,10 @@ describe(MetadataService.name, () => {await sut.handleMetadataExtraction({ id: assetStub.image.id });expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);- expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ latitude: null, longitude: null }));+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(+ expect.objectContaining({ latitude: null, longitude: null }),+ { lockedPropertiesBehavior: 'skip' },+ );});it('should extract tags from TagsList', async () => {@@ -571,6 +585,7 @@ describe(MetadataService.name, () => {expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.video.id);expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining({ orientation: ExifOrientation.Rotate270CW.toString() }),+ { lockedPropertiesBehavior: 'skip' },);});@@ -879,37 +894,40 @@ describe(MetadataService.name, () => {await sut.handleMetadataExtraction({ id: assetStub.image.id });expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(assetStub.image.id);- expect(mocks.asset.upsertExif).toHaveBeenCalledWith({- assetId: assetStub.image.id,- bitsPerSample: expect.any(Number),- autoStackId: null,- colorspace: tags.ColorSpace,- dateTimeOriginal: dateForTest,- description: tags.ImageDescription,- exifImageHeight: null,- exifImageWidth: null,- exposureTime: tags.ExposureTime,- fNumber: null,- fileSizeInByte: 123_456,- focalLength: tags.FocalLength,- fps: null,- iso: tags.ISO,- latitude: null,- lensModel: tags.LensModel,- livePhotoCID: tags.MediaGroupUUID,- longitude: null,- make: tags.Make,- model: tags.Model,- modifyDate: expect.any(Date),- orientation: tags.Orientation?.toString(),- profileDescription: tags.ProfileDescription,- projectionType: 'EQUIRECTANGULAR',- timeZone: tags.tz,- rating: tags.Rating,- country: null,- state: null,- city: null,- });+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(+ {+ assetId: assetStub.image.id,+ bitsPerSample: expect.any(Number),+ autoStackId: null,+ colorspace: tags.ColorSpace,+ dateTimeOriginal: dateForTest,+ description: tags.ImageDescription,+ exifImageHeight: null,+ exifImageWidth: null,+ exposureTime: tags.ExposureTime,+ fNumber: null,+ fileSizeInByte: 123_456,+ focalLength: tags.FocalLength,+ fps: null,+ iso: tags.ISO,+ latitude: null,+ lensModel: tags.LensModel,+ livePhotoCID: tags.MediaGroupUUID,+ longitude: null,+ make: tags.Make,+ model: tags.Model,+ modifyDate: expect.any(Date),+ orientation: tags.Orientation?.toString(),+ profileDescription: tags.ProfileDescription,+ projectionType: 'EQUIRECTANGULAR',+ timeZone: tags.tz,+ rating: tags.Rating,+ country: null,+ state: null,+ city: null,+ },+ { lockedPropertiesBehavior: 'skip' },+ );expect(mocks.asset.update).toHaveBeenCalledWith(expect.objectContaining({id: assetStub.image.id,@@ -943,6 +961,7 @@ describe(MetadataService.name, () => {expect.objectContaining({timeZone: 'UTC+0',}),+ { lockedPropertiesBehavior: 'skip' },);});@@ -1103,6 +1122,7 @@ describe(MetadataService.name, () => {expect.objectContaining({description: '',}),+ { lockedPropertiesBehavior: 'skip' },);mockReadTags({ ImageDescription: ' my\n description' });@@ -1111,6 +1131,7 @@ describe(MetadataService.name, () => {expect.objectContaining({description: 'my\n description',}),+ { lockedPropertiesBehavior: 'skip' },);});@@ -1123,6 +1144,7 @@ describe(MetadataService.name, () => {expect.objectContaining({description: '1000',}),+ { lockedPropertiesBehavior: 'skip' },);});@@ -1346,6 +1368,7 @@ describe(MetadataService.name, () => {expect.objectContaining({modifyDate: expect.any(Date),}),+ { lockedPropertiesBehavior: 'skip' },);});@@ -1358,6 +1381,7 @@ describe(MetadataService.name, () => {expect.objectContaining({rating: null,}),+ { lockedPropertiesBehavior: 'skip' },);});@@ -1370,6 +1394,7 @@ describe(MetadataService.name, () => {expect.objectContaining({rating: 5,}),+ { lockedPropertiesBehavior: 'skip' },);});@@ -1382,6 +1407,7 @@ describe(MetadataService.name, () => {expect.objectContaining({rating: -1,}),+ { lockedPropertiesBehavior: 'skip' },);});@@ -1503,7 +1529,9 @@ describe(MetadataService.name, () => {mockReadTags(exif);await sut.handleMetadataExtraction({ id: assetStub.image.id });- expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining(expected));+ expect(mocks.asset.upsertExif).toHaveBeenCalledWith(expect.objectContaining(expected), {+ lockedPropertiesBehavior: 'skip',+ });});it.each([@@ -1529,6 +1557,7 @@ describe(MetadataService.name, () => {expect.objectContaining({lensModel: expected,}),+ { lockedPropertiesBehavior: 'skip' },);});});@@ -1637,12 +1666,14 @@ describe(MetadataService.name, () => {describe('handleSidecarWrite', () => {it('should skip assets that no longer exist', async () => {+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(void 0);await expect(sut.handleSidecarWrite({ id: 'asset-123' })).resolves.toBe(JobStatus.Failed);expect(mocks.metadata.writeTags).not.toHaveBeenCalled();});it('should skip jobs with no metadata', async () => {+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([]);const asset = factory.jobAssets.sidecarWrite();mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(asset);await expect(sut.handleSidecarWrite({ id: asset.id })).resolves.toBe(JobStatus.Skipped);@@ -1655,20 +1686,22 @@ describe(MetadataService.name, () => {const gps = 12;const date = '2023-11-22T04:56:12.196Z';+ mocks.assetJob.getLockedPropertiesForMetadataExtraction.mockResolvedValue([+ 'description',+ 'latitude',+ 'longitude',+ 'dateTimeOriginal',+ ]);mocks.assetJob.getForSidecarWriteJob.mockResolvedValue(asset);await expect(sut.handleSidecarWrite({id: asset.id,- description,- latitude: gps,- longitude: gps,- dateTimeOriginal: date,}),).resolves.toBe(JobStatus.Success);expect(mocks.metadata.writeTags).toHaveBeenCalledWith(asset.files[0].path, {+ DateTimeOriginal: date,Description: description,ImageDescription: description,- DateTimeOriginal: date,GPSLatitude: gps,GPSLongitude: gps,});diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.tsindex a1267604b2..8da7c88ecc 100644--- a/server/src/services/metadata.service.ts+++ b/server/src/services/metadata.service.ts@@ -290,7 +290,7 @@ export class MetadataService extends BaseService {};const promises: Promise<unknown>[] = [- this.assetRepository.upsertExif(exifData),+ this.assetRepository.upsertExif(exifData, { lockedPropertiesBehavior: 'skip' }),this.assetRepository.update({id: asset.id,duration: this.getDuration(exifTags),@@ -393,22 +393,34 @@ export class MetadataService extends BaseService {@OnJob({ name: JobName.SidecarWrite, queue: QueueName.Sidecar })async handleSidecarWrite(job: JobOf<JobName.SidecarWrite>): Promise<JobStatus> {- const { id, description, dateTimeOriginal, latitude, longitude, rating, tags } = job;+ const { id, tags } = job;const asset = await this.assetJobRepository.getForSidecarWriteJob(id);if (!asset) {return JobStatus.Failed;}+ const lockedProperties = await this.assetJobRepository.getLockedPropertiesForMetadataExtraction(id);const tagsList = (asset.tags || []).map((tag) => tag.value);const { sidecarFile } = getAssetFiles(asset.files);const sidecarPath = sidecarFile?.path || `${asset.originalPath}.xmp`;+ const { description, dateTimeOriginal, latitude, longitude, rating } = _.pick(+ {+ description: asset.exifInfo.description,+ dateTimeOriginal: asset.exifInfo.dateTimeOriginal,+ latitude: asset.exifInfo.latitude,+ longitude: asset.exifInfo.longitude,+ rating: asset.exifInfo.rating,+ },+ lockedProperties,+ );+const exif = _.omitBy(<Tags>{Description: description,ImageDescription: description,- DateTimeOriginal: dateTimeOriginal,+ DateTimeOriginal: dateTimeOriginal ? String(dateTimeOriginal) : undefined,GPSLatitude: latitude,GPSLongitude: longitude,Rating: rating,diff --git a/server/src/types.ts b/server/src/types.tsindex a33dba490c..e404332fac 100644--- a/server/src/types.ts+++ b/server/src/types.ts@@ -222,11 +222,6 @@ export interface IDeleteFilesJob extends IBaseJob {}export interface ISidecarWriteJob extends IEntityJob {- description?: string;- dateTimeOriginal?: string;- latitude?: number;- longitude?: number;- rating?: number;tags?: true;}diff --git a/server/src/utils/database.ts b/server/src/utils/database.tsindex f8dbd5e78c..656e8e628a 100644--- a/server/src/utils/database.ts+++ b/server/src/utils/database.ts@@ -15,7 +15,7 @@ import { PostgresJSDialect } from 'kysely-postgres-js';import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';import { parse } from 'pg-connection-string';import postgres, { Notice, PostgresError } from 'postgres';-import { columns, Exif, Person } from 'src/database';+import { columns, Exif, lockableProperties, LockableProperty, Person } from 'src/database';import { AssetFileType, AssetVisibility, DatabaseExtension, DatabaseSslMode } from 'src/enum';import { AssetSearchBuilderOptions } from 'src/repositories/search.repository';import { DB } from 'src/schema';@@ -488,3 +488,10 @@ export function vectorIndexQuery({ vectorExtension, table, indexName, lists }: V}}}++export const updateLockedColumns = <T extends Record<string, unknown> & { lockedProperties?: LockableProperty[] }>(+ exif: T,+) => {+ exif.lockedProperties = lockableProperties.filter((property) => property in exif);+ return exif;+};diff --git a/server/test/medium.factory.ts b/server/test/medium.factory.tsindex efcdc59793..82ea2cd1fc 100644--- a/server/test/medium.factory.ts+++ b/server/test/medium.factory.ts@@ -202,7 +202,7 @@ export class MediumTestContext<S extends BaseService = BaseService> {}async newExif(dto: Insertable<AssetExifTable>) {- const result = await this.get(AssetRepository).upsertExif(dto);+ const result = await this.get(AssetRepository).upsertExif(dto, { lockedPropertiesBehavior: 'override' });return { result };}diff --git a/server/test/medium/specs/services/asset.service.spec.ts b/server/test/medium/specs/services/asset.service.spec.tsindex 13bc1ca9a9..8b54019fcf 100644--- a/server/test/medium/specs/services/asset.service.spec.ts+++ b/server/test/medium/specs/services/asset.service.spec.ts@@ -268,4 +268,65 @@ describe(AssetService.name, () => {});});});++ describe('update', () => {+ it('should update dateTimeOriginal', async () => {+ const { sut, ctx } = setup();+ ctx.getMock(JobRepository).queue.mockResolvedValue();+ const { user } = await ctx.newUser();+ const auth = factory.auth({ user });+ const { asset } = await ctx.newAsset({ ownerId: user.id });+ await ctx.newExif({ assetId: asset.id, description: 'test' });++ await expect(+ ctx.database+ .selectFrom('asset_exif')+ .select('lockedProperties')+ .where('assetId', '=', asset.id)+ .executeTakeFirstOrThrow(),+ ).resolves.toEqual({ lockedProperties: null });+ await sut.update(auth, asset.id, { dateTimeOriginal: '2023-11-19T18:11:00.000-07:00' });++ await expect(+ ctx.database+ .selectFrom('asset_exif')+ .select('lockedProperties')+ .where('assetId', '=', asset.id)+ .executeTakeFirstOrThrow(),+ ).resolves.toEqual({ lockedProperties: ['dateTimeOriginal'] });+ await expect(ctx.get(AssetRepository).getById(asset.id, { exifInfo: true })).resolves.toEqual(+ expect.objectContaining({+ exifInfo: expect.objectContaining({ dateTimeOriginal: '2023-11-20T01:11:00+00:00' }),+ }),+ );+ });+ });++ describe('updateAll', () => {+ it('should relatively update assets', async () => {+ const { sut, ctx } = setup();+ ctx.getMock(JobRepository).queueAll.mockResolvedValue();+ const { user } = await ctx.newUser();+ const auth = factory.auth({ user });+ const { asset } = await ctx.newAsset({ ownerId: user.id });+ await ctx.newExif({ assetId: asset.id, dateTimeOriginal: '2023-11-19T18:11:00' });++ await sut.updateAll(auth, { ids: [asset.id], dateTimeRelative: -11 });++ await expect(+ ctx.database+ .selectFrom('asset_exif')+ .select('lockedProperties')+ .where('assetId', '=', asset.id)+ .executeTakeFirstOrThrow(),+ ).resolves.toEqual({ lockedProperties: ['timeZone', 'dateTimeOriginal'] });+ await expect(ctx.get(AssetRepository).getById(asset.id, { exifInfo: true })).resolves.toEqual(+ expect.objectContaining({+ exifInfo: expect.objectContaining({+ dateTimeOriginal: '2023-11-19T18:00:00+00:00',+ }),+ }),+ );+ });+ });});diff --git a/server/test/medium/specs/services/metadata.service.spec.ts b/server/test/medium/specs/services/metadata.service.spec.tsindex 1d144e9c9c..4c3499857d 100644--- a/server/test/medium/specs/services/metadata.service.spec.ts+++ b/server/test/medium/specs/services/metadata.service.spec.ts@@ -95,6 +95,7 @@ describe(MetadataService.name, () => {dateTimeOriginal: new Date(expected.dateTimeOriginal),timeZone: expected.timeZone,}),+ { lockedPropertiesBehavior: 'skip' },);expect(mocks.asset.update).toHaveBeenCalledWith(diff --git a/server/test/medium/specs/sync/sync-album-asset-exif.spec.ts b/server/test/medium/specs/sync/sync-album-asset-exif.spec.tsindex fd563f4db1..1865fc2c80 100644--- a/server/test/medium/specs/sync/sync-album-asset-exif.spec.ts+++ b/server/test/medium/specs/sync/sync-album-asset-exif.spec.ts@@ -2,6 +2,7 @@ import { Kysely } from 'kysely';import { AlbumUserRole, SyncEntityType, SyncRequestType } from 'src/enum';import { AssetRepository } from 'src/repositories/asset.repository';import { DB } from 'src/schema';+import { updateLockedColumns } from 'src/utils/database';import { SyncTestContext } from 'test/medium.factory';import { factory } from 'test/small.factory';import { getKyselyDB, wait } from 'test/utils';@@ -288,10 +289,13 @@ describe(SyncRequestType.AlbumAssetExifsV1, () => {// update the assetconst assetRepository = ctx.get(AssetRepository);- await assetRepository.upsertExif({- assetId: asset.id,- city: 'New City',- });+ await assetRepository.upsertExif(+ updateLockedColumns({+ assetId: asset.id,+ city: 'New City',+ }),+ { lockedPropertiesBehavior: 'append' },+ );await expect(ctx.syncStream(auth, [SyncRequestType.AlbumAssetExifsV1])).resolves.toEqual([{@@ -346,10 +350,13 @@ describe(SyncRequestType.AlbumAssetExifsV1, () => {// update the assetconst assetRepository = ctx.get(AssetRepository);- await assetRepository.upsertExif({- assetId: assetDelayedExif.id,- city: 'Delayed Exif',- });+ await assetRepository.upsertExif(+ updateLockedColumns({+ assetId: assetDelayedExif.id,+ city: 'Delayed Exif',+ }),+ { lockedPropertiesBehavior: 'append' },+ );await expect(ctx.syncStream(auth, [SyncRequestType.AlbumAssetExifsV1])).resolves.toEqual([{diff --git a/server/test/small.factory.ts b/server/test/small.factory.tsindex 13cccce176..53d7c70794 100644--- a/server/test/small.factory.ts+++ b/server/test/small.factory.ts@@ -4,6 +4,7 @@ import {AuthApiKey,AuthSharedLink,AuthUser,+ Exif,Library,Memory,Partner,@@ -319,18 +320,28 @@ const versionHistoryFactory = () => ({version: '1.123.45',});-const assetSidecarWriteFactory = () => ({- id: newUuid(),- originalPath: '/path/to/original-path.jpg.xmp',- tags: [],- files: [- {- id: newUuid(),- path: '/path/to/original-path.jpg.xmp',- type: AssetFileType.Sidecar,- },- ],-});+const assetSidecarWriteFactory = () => {+ const id = newUuid();+ return {+ id,+ originalPath: '/path/to/original-path.jpg.xmp',+ tags: [],+ files: [+ {+ id: newUuid(),+ path: '/path/to/original-path.jpg.xmp',+ type: AssetFileType.Sidecar,+ },+ ],+ exifInfo: {+ assetId: id,+ description: 'this is a description',+ latitude: 12,+ longitude: 12,+ dateTimeOriginal: '2023-11-22T04:56:12.196Z',+ } as unknown as Exif,+ };+};const assetOcrFactory = (ocr: {