Skip to content

High performance self-hosted photo and video management solution.

feat(server): Nullable asset dates (#15669)

Jonathan Jogenfors5407a2853388
diff --git a/e2e/src/api/specs/library.e2e-spec.ts b/e2e/src/api/specs/library.e2e-spec.ts
index e2e69529fb..21fb945d1a 100644
--- a/e2e/src/api/specs/library.e2e-spec.ts
+++ b/e2e/src/api/specs/library.e2e-spec.ts
@@ -298,6 +298,7 @@ describe('/libraries', () => {
expect(status).toBe(204);
await utils.waitForQueueFinish(admin.accessToken, 'library');
+ await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
const { assets } = await utils.searchAssets(admin.accessToken, {
originalPath: `${testAssetDirInternal}/temp/directoryA/assetA.png`,
diff --git a/server/src/db.d.ts b/server/src/db.d.ts
index 648ccf4bcf..2e10e1aded 100644
--- a/server/src/db.d.ts
+++ b/server/src/db.d.ts
@@ -122,8 +122,8 @@ export interface Assets {
duplicateId: string | null;
duration: string | null;
encodedVideoPath: Generated<string | null>;
- fileCreatedAt: Timestamp;
- fileModifiedAt: Timestamp;
+ fileCreatedAt: Timestamp | null;
+ fileModifiedAt: Timestamp | null;
id: Generated<string>;
isArchived: Generated<boolean>;
isExternal: Generated<boolean>;
@@ -132,7 +132,7 @@ export interface Assets {
isVisible: Generated<boolean>;
libraryId: string | null;
livePhotoVideoId: string | null;
- localDateTime: Timestamp;
+ localDateTime: Timestamp | null;
originalFileName: string;
originalPath: string;
ownerId: string;
diff --git a/server/src/entities/asset.entity.ts b/server/src/entities/asset.entity.ts
index a90236de84..1a7d3ee1c1 100644
--- a/server/src/entities/asset.entity.ts
+++ b/server/src/entities/asset.entity.ts
@@ -100,13 +100,13 @@ export class AssetEntity {
deletedAt!: Date | null;
@Index('idx_asset_file_created_at')
- @Column({ type: 'timestamptz' })
+ @Column({ type: 'timestamptz', nullable: true, default: null })
fileCreatedAt!: Date;
- @Column({ type: 'timestamptz' })
+ @Column({ type: 'timestamptz', nullable: true, default: null })
localDateTime!: Date;
- @Column({ type: 'timestamptz' })
+ @Column({ type: 'timestamptz', nullable: true, default: null })
fileModifiedAt!: Date;
@Column({ type: 'boolean', default: false })
@@ -180,6 +180,12 @@ export class AssetEntity {
duplicateId!: string | null;
}
+export type AssetEntityPlaceholder = AssetEntity & {
+ fileCreatedAt: Date | null;
+ fileModifiedAt: Date | null;
+ localDateTime: Date | null;
+};
+
export function withExif<O>(qb: SelectQueryBuilder<DB, 'assets', O>) {
return qb.leftJoin('exif', 'assets.id', 'exif.assetId').select((eb) => eb.fn.toJson(eb.table('exif')).as('exifInfo'));
}
@@ -419,5 +425,8 @@ export function searchAssetBuilder(kysely: Kysely<DB>, options: AssetSearchBuild
)
.$if(!!options.withExif, withExifInner)
.$if(!!(options.withFaces || options.withPeople || options.personIds), (qb) => qb.select(withFacesAndPeople))
- .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null));
+ .$if(!options.withDeleted, (qb) => qb.where('assets.deletedAt', 'is', null))
+ .where('assets.fileCreatedAt', 'is not', null)
+ .where('assets.fileModifiedAt', 'is not', null)
+ .where('assets.localDateTime', 'is not', null);
}
diff --git a/server/src/migrations/1737845696644-NullableDates.ts b/server/src/migrations/1737845696644-NullableDates.ts
new file mode 100644
index 0000000000..8a08b985c5
--- /dev/null
+++ b/server/src/migrations/1737845696644-NullableDates.ts
@@ -0,0 +1,18 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class NullableDates1737845696644 implements MigrationInterface {
+ name = 'NullableDates1737845696644'
+
+ public async up(queryRunner: QueryRunner): Promise<void> {
+ await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "fileCreatedAt" DROP NOT NULL`);
+ await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "localDateTime" DROP NOT NULL`);
+ await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "fileModifiedAt" DROP NOT NULL`);
+ }
+
+ public async down(queryRunner: QueryRunner): Promise<void> {
+ await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "fileModifiedAt" SET NOT NULL`);
+ await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "localDateTime" SET NOT NULL`);
+ await queryRunner.query(`ALTER TABLE "assets" ALTER COLUMN "fileCreatedAt" SET NOT NULL`);
+ }
+
+}
diff --git a/server/src/queries/activity.repository.sql b/server/src/queries/activity.repository.sql
index 8e9bb11f25..0ddb91c692 100644
--- a/server/src/queries/activity.repository.sql
+++ b/server/src/queries/activity.repository.sql
@@ -43,3 +43,6 @@ where
and "activity"."albumId" = $2
and "activity"."isLiked" = $3
and "assets"."deletedAt" is null
+ and "assets"."fileCreatedAt" is not null
+ and "assets"."fileModifiedAt" is not null
+ and "assets"."localDateTime" is not null
diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql
index 437e1e173c..3efc560b3b 100644
--- a/server/src/queries/asset.repository.sql
+++ b/server/src/queries/asset.repository.sql
@@ -159,6 +159,9 @@ where
"ownerId" = $1::uuid
and "deviceId" = $2
and "isVisible" = $3
+ and "assets"."fileCreatedAt" is not null
+ and "assets"."fileModifiedAt" is not null
+ and "assets"."localDateTime" is not null
and "deletedAt" is null
-- AssetRepository.getLivePhotoCount
@@ -260,6 +263,9 @@ with
where
"assets"."deletedAt" is null
and "assets"."isVisible" = $2
+ and "assets"."fileCreatedAt" is not null
+ and "assets"."fileModifiedAt" is not null
+ and "assets"."localDateTime" is not null
)
select
"timeBucket",
diff --git a/server/src/queries/search.repository.sql b/server/src/queries/search.repository.sql
index 72e8a6941d..9400700e56 100644
--- a/server/src/queries/search.repository.sql
+++ b/server/src/queries/search.repository.sql
@@ -13,6 +13,9 @@ where
and "assets"."isFavorite" = $4
and "assets"."isArchived" = $5
and "assets"."deletedAt" is null
+ and "assets"."fileCreatedAt" is not null
+ and "assets"."fileModifiedAt" is not null
+ and "assets"."localDateTime" is not null
order by
"assets"."fileCreatedAt" desc
limit
@@ -34,6 +37,9 @@ offset
and "assets"."isFavorite" = $4
and "assets"."isArchived" = $5
and "assets"."deletedAt" is null
+ and "assets"."fileCreatedAt" is not null
+ and "assets"."fileModifiedAt" is not null
+ and "assets"."localDateTime" is not null
and "assets"."id" < $6
order by
random()
@@ -54,6 +60,9 @@ union all
and "assets"."isFavorite" = $11
and "assets"."isArchived" = $12
and "assets"."deletedAt" is null
+ and "assets"."fileCreatedAt" is not null
+ and "assets"."fileModifiedAt" is not null
+ and "assets"."localDateTime" is not null
and "assets"."id" > $13
order by
random()
@@ -77,6 +86,9 @@ where
and "assets"."isFavorite" = $4
and "assets"."isArchived" = $5
and "assets"."deletedAt" is null
+ and "assets"."fileCreatedAt" is not null
+ and "assets"."fileModifiedAt" is not null
+ and "assets"."localDateTime" is not null
order by
smart_search.embedding <=> $6
limit
diff --git a/server/src/queries/view.repository.sql b/server/src/queries/view.repository.sql
index b368684cae..daa4159ea0 100644
--- a/server/src/queries/view.repository.sql
+++ b/server/src/queries/view.repository.sql
@@ -10,6 +10,9 @@ where
and "isVisible" = $3
and "isArchived" = $4
and "deletedAt" is null
+ and "fileModifiedAt" is not null
+ and "fileModifiedAt" is not null
+ and "localDateTime" is not null
-- ViewRepository.getAssetsByOriginalPath
select
@@ -23,6 +26,9 @@ where
and "isVisible" = $2
and "isArchived" = $3
and "deletedAt" is null
+ and "fileModifiedAt" is not null
+ and "fileModifiedAt" is not null
+ and "localDateTime" is not null
and "originalPath" like $4
and "originalPath" not like $5
order by
diff --git a/server/src/repositories/activity.repository.ts b/server/src/repositories/activity.repository.ts
index 99d3192341..d998fea23c 100644
--- a/server/src/repositories/activity.repository.ts
+++ b/server/src/repositories/activity.repository.ts
@@ -65,6 +65,9 @@ export class ActivityRepository {
.where('activity.albumId', '=', albumId)
.where('activity.isLiked', '=', false)
.where('assets.deletedAt', 'is', null)
+ .where('assets.fileCreatedAt', 'is not', null)
+ .where('assets.fileModifiedAt', 'is not', null)
+ .where('assets.localDateTime', 'is not', null)
.executeTakeFirstOrThrow();
return count as number;
diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts
index bd82cc0724..123116c62f 100644
--- a/server/src/repositories/asset.repository.ts
+++ b/server/src/repositories/asset.repository.ts
@@ -7,6 +7,7 @@ import { AssetFiles, AssetJobStatus, Assets, DB, Exif } from 'src/db';
import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
import {
AssetEntity,
+ AssetEntityPlaceholder,
hasPeople,
searchAssetBuilder,
truncatedDate,
@@ -183,8 +184,12 @@ export class AssetRepository {
.execute();
}
- create(asset: Insertable<Assets>): Promise<AssetEntity> {
- return this.db.insertInto('assets').values(asset).returningAll().executeTakeFirst() as any as Promise<AssetEntity>;
+ create(asset: Insertable<Assets>): Promise<AssetEntityPlaceholder> {
+ return this.db
+ .insertInto('assets')
+ .values(asset)
+ .returningAll()
+ .executeTakeFirst() as any as Promise<AssetEntityPlaceholder>;
}
@GenerateSql({ params: [DummyValue.UUID, { day: 1, month: 1 }] })
@@ -395,6 +400,9 @@ export class AssetRepository {
.where('ownerId', '=', asUuid(ownerId))
.where('deviceId', '=', deviceId)
.where('isVisible', '=', true)
+ .where('assets.fileCreatedAt', 'is not', null)
+ .where('assets.fileModifiedAt', 'is not', null)
+ .where('assets.localDateTime', 'is not', null)
.where('deletedAt', 'is', null)
.execute();
@@ -562,7 +570,10 @@ export class AssetRepository {
.where('job_status.duplicatesDetectedAt', 'is', null)
.where('job_status.previewAt', 'is not', null)
.where((eb) => eb.exists(eb.selectFrom('smart_search').where('assetId', '=', eb.ref('assets.id'))))
- .where('assets.isVisible', '=', true),
+ .where('assets.isVisible', '=', true)
+ .where('assets.fileCreatedAt', 'is not', null)
+ .where('assets.fileModifiedAt', 'is not', null)
+ .where('assets.localDateTime', 'is not', null),
)
.$if(property === WithoutProperty.ENCODED_VIDEO, (qb) =>
qb
@@ -656,6 +667,9 @@ export class AssetRepository {
.select((eb) => eb.fn.countAll().filterWhere('type', '=', AssetType.VIDEO).as(AssetType.VIDEO))
.select((eb) => eb.fn.countAll().filterWhere('type', '=', AssetType.OTHER).as(AssetType.OTHER))
.where('ownerId', '=', asUuid(ownerId))
+ .where('assets.fileCreatedAt', 'is not', null)
+ .where('assets.fileModifiedAt', 'is not', null)
+ .where('assets.localDateTime', 'is not', null)
.where('isVisible', '=', true)
.$if(isArchived !== undefined, (qb) => qb.where('isArchived', '=', isArchived!))
.$if(isFavorite !== undefined, (qb) => qb.where('isFavorite', '=', isFavorite!))
@@ -688,6 +702,9 @@ export class AssetRepository {
.$if(!!options.isTrashed, (qb) => qb.where('assets.status', '!=', AssetStatus.DELETED))
.where('assets.deletedAt', options.isTrashed ? 'is not' : 'is', null)
.where('assets.isVisible', '=', true)
+ .where('assets.fileCreatedAt', 'is not', null)
+ .where('assets.fileModifiedAt', 'is not', null)
+ .where('assets.localDateTime', 'is not', null)
.$if(!!options.albumId, (qb) =>
qb
.innerJoin('albums_assets_assets', 'assets.id', 'albums_assets_assets.assetsId')
diff --git a/server/src/repositories/view-repository.ts b/server/src/repositories/view-repository.ts
index f24b1bac6e..4fa670339e 100644
--- a/server/src/repositories/view-repository.ts
+++ b/server/src/repositories/view-repository.ts
@@ -18,6 +18,9 @@ export class ViewRepository {
.where('isVisible', '=', true)
.where('isArchived', '=', false)
.where('deletedAt', 'is', null)
+ .where('fileModifiedAt', 'is not', null)
+ .where('fileModifiedAt', 'is not', null)
+ .where('localDateTime', 'is not', null)
.execute();
return results.map((row) => row.directoryPath.replaceAll(/^\/|\/$/g, ''));
@@ -35,6 +38,9 @@ export class ViewRepository {
.where('isVisible', '=', true)
.where('isArchived', '=', false)
.where('deletedAt', 'is', null)
+ .where('fileModifiedAt', 'is not', null)
+ .where('fileModifiedAt', 'is not', null)
+ .where('localDateTime', 'is not', null)
.where('originalPath', 'like', `%${normalizedPath}/%`)
.where('originalPath', 'not like', `%${normalizedPath}/%/%`)
.orderBy(
diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts
index 4a9614e010..441d130c12 100644
--- a/server/src/services/library.service.ts
+++ b/server/src/services/library.service.ts
@@ -503,7 +503,7 @@ export class LibraryService extends BaseService {
}
const mtime = stat.mtime;
- const isAssetModified = mtime.toISOString() !== asset.fileModifiedAt.toISOString();
+ const isAssetModified = !asset.fileModifiedAt || mtime.toISOString() !== asset.fileModifiedAt.toISOString();
if (asset.isOffline || isAssetModified) {
this.logger.debug(`Asset was offline or modified, updating asset record ${asset.originalPath}`);
diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts
index 8c2e3646a0..19c3656e01 100644
--- a/server/src/services/metadata.service.ts
+++ b/server/src/services/metadata.service.ts
@@ -171,6 +171,14 @@ export class MetadataService extends BaseService {
this.logger.verbose('Exif Tags', exifTags);
+ if (!asset.fileCreatedAt) {
+ asset.fileCreatedAt = stats.mtime;
+ }
+
+ if (!asset.fileModifiedAt) {
+ asset.fileModifiedAt = stats.mtime;
+ }
+
const { dateTimeOriginal, localDateTime, timeZone, modifyDate } = this.getDates(asset, exifTags);
const { latitude, longitude, country, state, city } = await this.getGeo(exifTags, reverseGeocoding);