High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat(server): Nullable asset dates (#15669)
Jonathan Jogenfors5407a2853388
diff --git a/e2e/src/api/specs/library.e2e-spec.ts b/e2e/src/api/specs/library.e2e-spec.tsindex e2e69529fb..21fb945d1a 100644--- a/e2e/src/api/specs/library.e2e-spec.ts+++ b/e2e/src/api/specs/library.e2e-spec.ts@@ -298,6 +298,7 @@ describe('/libraries', () => {expect(status).toBe(204);await utils.waitForQueueFinish(admin.accessToken, 'library');+ await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');const { assets } = await utils.searchAssets(admin.accessToken, {originalPath: `${testAssetDirInternal}/temp/directoryA/assetA.png`,diff --git a/server/src/db.d.ts b/server/src/db.d.tsindex 648ccf4bcf..2e10e1aded 100644--- a/server/src/db.d.ts+++ b/server/src/db.d.ts@@ -122,8 +122,8 @@ export interface Assets {duplicateId: string | null;duration: string | null;encodedVideoPath: Generated<string | null>;- fileCreatedAt: Timestamp;- fileModifiedAt: Timestamp;+ fileCreatedAt: Timestamp | null;+ fileModifiedAt: Timestamp | null;id: Generated<string>;isArchived: Generated<boolean>;isExternal: Generated<boolean>;@@ -132,7 +132,7 @@ export interface Assets {isVisible: Generated<boolean>;libraryId: string | null;livePhotoVideoId: string | null;- localDateTime: Timestamp;+ localDateTime: Timestamp | null;originalFileName: string;originalPath: string;ownerId: string;diff --git a/server/src/entities/asset.entity.ts b/server/src/entities/asset.entity.tsindex a90236de84..1a7d3ee1c1 100644--- a/server/src/entities/asset.entity.ts+++ b/server/src/entities/asset.entity.ts@@ -100,13 +100,13 @@ export class AssetEntity {deletedAt!: Date | null;@Index('idx_asset_file_created_at')- @Column({ type: 'timestamptz' })+ @Column({ type: 'timestamptz', nullable: true, default: null })fileCreatedAt!: Date;- @Column({ type: 'timestamptz' })+ @Column({ type: 'timestamptz', nullable: true, default: null })localDateTime!: Date;- @Column({ type: 'timestamptz' })+ @Column({ type: 'timestamptz', nullable: true, default: null })fileModifiedAt!: Date;@Column({ type: 'boolean', default: false })@@ -180,6 +180,12 @@ export class AssetEntity {duplicateId!: string | null;}+export type AssetEntityPlaceholder = AssetEntity & {+ fileCreatedAt: Date | null;+ fileModifiedAt: Date | null;+ localDateTime: Date | null;+};+export function withExif<O>(qb: SelectQueryBuilder<DB, 'assets', O>) {return qb.leftJoin('exif', 'assets.id', 'exif.assetId').select((eb) => eb.fn.toJson(eb.table('exif')).as('exifInfo'));}@@ -419,5 +425,8 @@ export function searchAssetBuilder(kysely: Kysely<DB>, options: AssetSearchBuild).$if(!!options.withExif, withExifInner).$if(!!(options.withFaces || options.withPeople || options.personIds), (qb) => qb.select(withFacesAndPeople))- .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null));+ .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null))+ .where('assets.fileCreatedAt', 'is not', null)+ .where('assets.fileModifiedAt', 'is not', null)+ .where('assets.localDateTime', 'is not', null);}diff --git a/server/src/migrations/1737845696644-NullableDates.ts b/server/src/migrations/1737845696644-NullableDates.tsnew file mode 100644index 0000000000..8a08b985c5--- /dev/null+++ b/server/src/migrations/1737845696644-NullableDates.ts@@ -0,0 +1,18 @@+import { MigrationInterface, QueryRunner } from "typeorm";++export class NullableDates1737845696644 implements MigrationInterface {+ name = 'NullableDates1737845696644'++ public async up(queryRunner: QueryRunner): Promise<void> {+ await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "fileCreatedAt" DROP NOT NULL`);+ await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "localDateTime" DROP NOT NULL`);+ await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "fileModifiedAt" DROP NOT NULL`);+ }++ public async down(queryRunner: QueryRunner): Promise<void> {+ await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "fileModifiedAt" SET NOT NULL`);+ await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "localDateTime" SET NOT NULL`);+ await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "fileCreatedAt" SET NOT NULL`);+ }++}diff --git a/server/src/queries/activity.repository.sql b/server/src/queries/activity.repository.sqlindex 8e9bb11f25..0ddb91c692 100644--- a/server/src/queries/activity.repository.sql+++ b/server/src/queries/activity.repository.sql@@ -43,3 +43,6 @@ whereand "activity"."albumId" = $2and "activity"."isLiked" = $3and "assets"."deletedAt" is null+ and "assets"."fileCreatedAt" is not null+ and "assets"."fileModifiedAt" is not null+ and "assets"."localDateTime" is not nulldiff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sqlindex 437e1e173c..3efc560b3b 100644--- a/server/src/queries/asset.repository.sql+++ b/server/src/queries/asset.repository.sql@@ -159,6 +159,9 @@ where"ownerId" = $1::uuidand "deviceId" = $2and "isVisible" = $3+ and "assets"."fileCreatedAt" is not null+ and "assets"."fileModifiedAt" is not null+ and "assets"."localDateTime" is not nulland "deletedAt" is null-- AssetRepository.getLivePhotoCount@@ -260,6 +263,9 @@ withwhere"assets"."deletedAt" is nulland "assets"."isVisible" = $2+ and "assets"."fileCreatedAt" is not null+ and "assets"."fileModifiedAt" is not null+ and "assets"."localDateTime" is not null)select"timeBucket",diff --git a/server/src/queries/search.repository.sql b/server/src/queries/search.repository.sqlindex 72e8a6941d..9400700e56 100644--- a/server/src/queries/search.repository.sql+++ b/server/src/queries/search.repository.sql@@ -13,6 +13,9 @@ whereand "assets"."isFavorite" = $4and "assets"."isArchived" = $5and "assets"."deletedAt" is null+ and "assets"."fileCreatedAt" is not null+ and "assets"."fileModifiedAt" is not null+ and "assets"."localDateTime" is not nullorder by"assets"."fileCreatedAt" desclimit@@ -34,6 +37,9 @@ offsetand "assets"."isFavorite" = $4and "assets"."isArchived" = $5and "assets"."deletedAt" is null+ and "assets"."fileCreatedAt" is not null+ and "assets"."fileModifiedAt" is not null+ and "assets"."localDateTime" is not nulland "assets"."id" < $6order byrandom()@@ -54,6 +60,9 @@ union alland "assets"."isFavorite" = $11and "assets"."isArchived" = $12and "assets"."deletedAt" is null+ and "assets"."fileCreatedAt" is not null+ and "assets"."fileModifiedAt" is not null+ and "assets"."localDateTime" is not nulland "assets"."id" > $13order byrandom()@@ -77,6 +86,9 @@ whereand "assets"."isFavorite" = $4and "assets"."isArchived" = $5and "assets"."deletedAt" is null+ and "assets"."fileCreatedAt" is not null+ and "assets"."fileModifiedAt" is not null+ and "assets"."localDateTime" is not nullorder bysmart_search.embedding <=> $6limitdiff --git a/server/src/queries/view.repository.sql b/server/src/queries/view.repository.sqlindex b368684cae..daa4159ea0 100644--- a/server/src/queries/view.repository.sql+++ b/server/src/queries/view.repository.sql@@ -10,6 +10,9 @@ whereand "isVisible" = $3and "isArchived" = $4and "deletedAt" is null+ and "fileModifiedAt" is not null+ and "fileModifiedAt" is not null+ and "localDateTime" is not null-- ViewRepository.getAssetsByOriginalPathselect@@ -23,6 +26,9 @@ whereand "isVisible" = $2and "isArchived" = $3and "deletedAt" is null+ and "fileModifiedAt" is not null+ and "fileModifiedAt" is not null+ and "localDateTime" is not nulland "originalPath" like $4and "originalPath" not like $5order bydiff --git a/server/src/repositories/activity.repository.ts b/server/src/repositories/activity.repository.tsindex 99d3192341..d998fea23c 100644--- a/server/src/repositories/activity.repository.ts+++ b/server/src/repositories/activity.repository.ts@@ -65,6 +65,9 @@ export class ActivityRepository {.where('activity.albumId', '=', albumId).where('activity.isLiked', '=', false).where('assets.deletedAt', 'is', null)+ .where('assets.fileCreatedAt', 'is not', null)+ .where('assets.fileModifiedAt', 'is not', null)+ .where('assets.localDateTime', 'is not', null).executeTakeFirstOrThrow();return count as number;diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.tsindex bd82cc0724..123116c62f 100644--- a/server/src/repositories/asset.repository.ts+++ b/server/src/repositories/asset.repository.ts@@ -7,6 +7,7 @@ import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db';import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';import {AssetEntity,+ AssetEntityPlaceholder,hasPeople,searchAssetBuilder,truncatedDate,@@ -183,8 +184,12 @@ export class AssetRepository {.execute();}- create(asset: Insertable<Assets>): Promise<AssetEntity> {- return this.db.insertInto('assets').values(asset).returningAll().executeTakeFirst() as any as Promise<AssetEntity>;+ create(asset: Insertable<Assets>): Promise<AssetEntityPlaceholder> {+ return this.db+ .insertInto('assets')+ .values(asset)+ .returningAll()+ .executeTakeFirst() as any as Promise<AssetEntityPlaceholder>;}@GenerateSql({ params: [DummyValue.UUID, { day: 1, month: 1 }] })@@ -395,6 +400,9 @@ export class AssetRepository {.where('ownerId', '=', asUuid(ownerId)).where('deviceId', '=', deviceId).where('isVisible', '=', true)+ .where('assets.fileCreatedAt', 'is not', null)+ .where('assets.fileModifiedAt', 'is not', null)+ .where('assets.localDateTime', 'is not', null).where('deletedAt', 'is', null).execute();@@ -562,7 +570,10 @@ export class AssetRepository {.where('job_status.duplicatesDetectedAt', 'is', null).where('job_status.previewAt', 'is not', null).where((eb) => eb.exists(eb.selectFrom('smart_search').where('assetId', '=', eb.ref('assets.id'))))- .where('assets.isVisible', '=', true),+ .where('assets.isVisible', '=', true)+ .where('assets.fileCreatedAt', 'is not', null)+ .where('assets.fileModifiedAt', 'is not', null)+ .where('assets.localDateTime', 'is not', null),).$if(property === WithoutProperty.ENCODED_VIDEO, (qb) =>qb@@ -656,6 +667,9 @@ export class AssetRepository {.select((eb) => eb.fn.countAll().filterWhere('type', '=', AssetType.VIDEO).as(AssetType.VIDEO)).select((eb) => eb.fn.countAll().filterWhere('type', '=', AssetType.OTHER).as(AssetType.OTHER)).where('ownerId', '=', asUuid(ownerId))+ .where('assets.fileCreatedAt', 'is not', null)+ .where('assets.fileModifiedAt', 'is not', null)+ .where('assets.localDateTime', 'is not', null).where('isVisible', '=', true).$if(isArchived !== undefined, (qb) => qb.where('isArchived', '=', isArchived!)).$if(isFavorite !== undefined, (qb) => qb.where('isFavorite', '=', isFavorite!))@@ -688,6 +702,9 @@ export class AssetRepository {.$if(!!options.isTrashed, (qb) => qb.where('assets.status', '!=', AssetStatus.DELETED)).where('assets.deletedAt', options.isTrashed ? 'is not' : 'is', null).where('assets.isVisible', '=', true)+ .where('assets.fileCreatedAt', 'is not', null)+ .where('assets.fileModifiedAt', 'is not', null)+ .where('assets.localDateTime', 'is not', null).$if(!!options.albumId, (qb) =>qb.innerJoin('albums_assets_assets', 'assets.id', 'albums_assets_assets.assetsId')diff --git a/server/src/repositories/view-repository.ts b/server/src/repositories/view-repository.tsindex f24b1bac6e..4fa670339e 100644--- a/server/src/repositories/view-repository.ts+++ b/server/src/repositories/view-repository.ts@@ -18,6 +18,9 @@ export class ViewRepository {.where('isVisible', '=', true).where('isArchived', '=', false).where('deletedAt', 'is', null)+ .where('fileModifiedAt', 'is not', null)+ .where('fileModifiedAt', 'is not', null)+ .where('localDateTime', 'is not', null).execute();return results.map((row) => row.directoryPath.replaceAll(/^\/|\/$/g, ''));@@ -35,6 +38,9 @@ export class ViewRepository {.where('isVisible', '=', true).where('isArchived', '=', false).where('deletedAt', 'is', null)+ .where('fileModifiedAt', 'is not', null)+ .where('fileModifiedAt', 'is not', null)+ .where('localDateTime', 'is not', null).where('originalPath', 'like', `%${normalizedPath}/%`).where('originalPath', 'not like', `%${normalizedPath}/%/%`).orderBy(diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.tsindex 4a9614e010..441d130c12 100644--- a/server/src/services/library.service.ts+++ b/server/src/services/library.service.ts@@ -503,7 +503,7 @@ export class LibraryService extends BaseService {}const mtime = stat.mtime;- const isAssetModified = mtime.toISOString() !== asset.fileModifiedAt.toISOString();+ const isAssetModified = !asset.fileModifiedAt || mtime.toISOString() !== asset.fileModifiedAt.toISOString();if (asset.isOffline || isAssetModified) {this.logger.debug(`Asset was offline or modified, updating asset record ${asset.originalPath}`);diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.tsindex 8c2e3646a0..19c3656e01 100644--- a/server/src/services/metadata.service.ts+++ b/server/src/services/metadata.service.ts@@ -171,6 +171,14 @@ export class MetadataService extends BaseService {this.logger.verbose('Exif Tags', exifTags);+ if (!asset.fileCreatedAt) {+ asset.fileCreatedAt = stats.mtime;+ }++ if (!asset.fileModifiedAt) {+ asset.fileModifiedAt = stats.mtime;+ }+const { dateTimeOriginal, localDateTime, timeZone, modifyDate } = this.getDates(asset, exifTags);const { latitude, longitude, country, state, city } = await this.getGeo(exifTags, reverseGeocoding);