Skip to content

High performance self-hosted photo and video management solution.

feat(server): add checksum algorithm field (#26573)

Jonathan Jogenfors22bf7c200535
diff --git a/server/src/database.ts b/server/src/database.ts
index f4878b2cee..4f339624e6 100644
--- a/server/src/database.ts
+++ b/server/src/database.ts
@@ -5,6 +5,7 @@ import {
AssetFileType,
AssetType,
AssetVisibility,
+ ChecksumAlgorithm,
MemoryType,
Permission,
PluginContext,
@@ -112,6 +113,7 @@ export type Memory = {
export type Asset = {
id: string;
checksum: Buffer<ArrayBufferLike>;
+ checksumAlgorithm: ChecksumAlgorithm;
deviceAssetId: string;
deviceId: string;
fileCreatedAt: Date;
@@ -330,6 +332,7 @@ export const columns = {
asset: [
'asset.id',
'asset.checksum',
+ 'asset.checksumAlgorithm',
'asset.deviceAssetId',
'asset.deviceId',
'asset.fileCreatedAt',
diff --git a/server/src/dtos/asset-response.dto.ts b/server/src/dtos/asset-response.dto.ts
index 8b38b2e124..2c2f57bbb2 100644
--- a/server/src/dtos/asset-response.dto.ts
+++ b/server/src/dtos/asset-response.dto.ts
@@ -13,7 +13,7 @@ import {
} from 'src/dtos/person.dto';
import { TagResponseDto, mapTag } from 'src/dtos/tag.dto';
import { UserResponseDto, mapUser } from 'src/dtos/user.dto';
-import { AssetStatus, AssetType, AssetVisibility } from 'src/enum';
+import { AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum';
import { ImageDimensions, MaybeDehydrated } from 'src/types';
import { getDimensions } from 'src/utils/asset.util';
import { hexOrBufferToBase64 } from 'src/utils/bytes';
@@ -148,6 +148,7 @@ export type MapAsset = {
updateId: string;
status: AssetStatus;
checksum: Buffer<ArrayBufferLike>;
+ checksumAlgorithm: ChecksumAlgorithm;
deviceAssetId: string;
deviceId: string;
duplicateId: string | null;
diff --git a/server/src/enum.ts b/server/src/enum.ts
index 60f45efd6e..1277a39036 100644
--- a/server/src/enum.ts
+++ b/server/src/enum.ts
@@ -37,6 +37,11 @@ export enum AssetType {
Other = 'OTHER',
}
+export enum ChecksumAlgorithm {
+ sha1File = 'sha1', // sha1 checksum of the whole file contents
+ sha1Path = 'sha1-path', // sha1 checksum of "path:" plus the file path, currently used in external libraries, deprecated
+}
+
export enum AssetFileType {
/**
* An full/large-size image extracted/converted from RAW photos
diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sql
index cf5b8f02dc..554aa3b195 100644
--- a/server/src/queries/asset.job.repository.sql
+++ b/server/src/queries/asset.job.repository.sql
@@ -249,6 +249,7 @@ where
select
"asset"."id",
"asset"."checksum",
+ "asset"."checksumAlgorithm",
"asset"."deviceAssetId",
"asset"."deviceId",
"asset"."fileCreatedAt",
diff --git a/server/src/schema/enums.ts b/server/src/schema/enums.ts
index c68f152779..f63a09c462 100644
--- a/server/src/schema/enums.ts
+++ b/server/src/schema/enums.ts
@@ -1,5 +1,5 @@
import { registerEnum } from '@immich/sql-tools';
-import { AssetStatus, AssetVisibility, SourceType } from 'src/enum';
+import { AssetStatus, AssetVisibility, ChecksumAlgorithm, SourceType } from 'src/enum';
export const assets_status_enum = registerEnum({
name: 'assets_status_enum',
@@ -15,3 +15,8 @@ export const asset_visibility_enum = registerEnum({
name: 'asset_visibility_enum',
values: Object.values(AssetVisibility),
});
+
+export const asset_checksum_algorithm_enum = registerEnum({
+ name: 'asset_checksum_algorithm_enum',
+ values: Object.values(ChecksumAlgorithm),
+});
diff --git a/server/src/schema/migrations/1774548649115-AddChecksumAlgorithm.ts.ts b/server/src/schema/migrations/1774548649115-AddChecksumAlgorithm.ts.ts
new file mode 100644
index 0000000000..477e7d0e48
--- /dev/null
+++ b/server/src/schema/migrations/1774548649115-AddChecksumAlgorithm.ts.ts
@@ -0,0 +1,21 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+ await sql`CREATE TYPE "asset_checksum_algorithm_enum" AS ENUM ('sha1','sha1-path');`.execute(db);
+ await sql`ALTER TABLE "asset" ADD "checksumAlgorithm" asset_checksum_algorithm_enum;`.execute(db);
+
+ await sql`
+ UPDATE "asset"
+ SET "checksumAlgorithm" = CASE
+ WHEN "isExternal" = true THEN 'sha1-path'::asset_checksum_algorithm_enum
+ ELSE 'sha1'::asset_checksum_algorithm_enum
+ END
+ `.execute(db);
+
+ await sql`ALTER TABLE "asset" ALTER COLUMN "checksumAlgorithm" SET NOT NULL;`.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+ await sql`ALTER TABLE "asset" DROP COLUMN "checksumAlgorithm";`.execute(db);
+ await sql`DROP TYPE "asset_checksum_algorithm_enum";`.execute(db);
+}
diff --git a/server/src/schema/tables/asset.table.ts b/server/src/schema/tables/asset.table.ts
index 8bdaa59bc6..7418dab102 100644
--- a/server/src/schema/tables/asset.table.ts
+++ b/server/src/schema/tables/asset.table.ts
@@ -12,8 +12,8 @@ import {
UpdateDateColumn,
} from '@immich/sql-tools';
import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators';
-import { AssetStatus, AssetType, AssetVisibility } from 'src/enum';
-import { asset_visibility_enum, assets_status_enum } from 'src/schema/enums';
+import { AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum';
+import { asset_checksum_algorithm_enum, asset_visibility_enum, assets_status_enum } from 'src/schema/enums';
import { asset_delete_audit } from 'src/schema/functions';
import { LibraryTable } from 'src/schema/tables/library.table';
import { StackTable } from 'src/schema/tables/stack.table';
@@ -95,6 +95,9 @@ export class AssetTable {
@Column({ type: 'bytea', index: true })
checksum!: Buffer; // sha1 checksum
+ @Column({ enum: asset_checksum_algorithm_enum })
+ checksumAlgorithm!: ChecksumAlgorithm;
+
@ForeignKeyColumn(() => AssetTable, { nullable: true, onUpdate: 'CASCADE', onDelete: 'SET NULL' })
livePhotoVideoId!: string | null;
diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.ts
index 03677054ec..8b8efd19c5 100644
--- a/server/src/services/asset-media.service.ts
+++ b/server/src/services/asset-media.service.ts
@@ -27,6 +27,7 @@ import {
AssetStatus,
AssetVisibility,
CacheControl,
+ ChecksumAlgorithm,
JobName,
Permission,
StorageFolder,
@@ -425,6 +426,7 @@ export class AssetMediaService extends BaseService {
deviceId: asset.deviceId,
type: asset.type,
checksum: asset.checksum,
+ checksumAlgorithm: asset.checksumAlgorithm,
fileCreatedAt: asset.fileCreatedAt,
localDateTime: asset.localDateTime,
fileModifiedAt: asset.fileModifiedAt,
@@ -446,6 +448,7 @@ export class AssetMediaService extends BaseService {
libraryId: null,
checksum: file.checksum,
+ checksumAlgorithm: ChecksumAlgorithm.sha1File,
originalPath: file.originalPath,
deviceAssetId: dto.deviceAssetId,
diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts
index 841fa4743c..9f2d69bab5 100644
--- a/server/src/services/library.service.ts
+++ b/server/src/services/library.service.ts
@@ -17,7 +17,17 @@ import {
ValidateLibraryImportPathResponseDto,
ValidateLibraryResponseDto,
} from 'src/dtos/library.dto';
-import { AssetStatus, AssetType, CronJob, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum';
+import {
+ AssetStatus,
+ AssetType,
+ ChecksumAlgorithm,
+ CronJob,
+ DatabaseLock,
+ ImmichWorker,
+ JobName,
+ JobStatus,
+ QueueName,
+} from 'src/enum';
import { ArgOf } from 'src/repositories/event.repository';
import { AssetSyncResult } from 'src/repositories/library.repository';
import { AssetTable } from 'src/schema/tables/asset.table';
@@ -400,6 +410,7 @@ export class LibraryService extends BaseService {
ownerId,
libraryId,
checksum: this.cryptoRepository.hashSha1(`path:${assetPath}`),
+ checksumAlgorithm: ChecksumAlgorithm.sha1Path,
originalPath: assetPath,
fileCreatedAt: stat.mtime,
diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts
index cb35e21d0a..f9d17079e3 100644
--- a/server/src/services/metadata.service.spec.ts
+++ b/server/src/services/metadata.service.spec.ts
@@ -7,6 +7,7 @@ import {
AssetFileType,
AssetType,
AssetVisibility,
+ ChecksumAlgorithm,
ExifOrientation,
ImmichWorker,
JobName,
@@ -652,6 +653,7 @@ describe(MetadataService.name, () => {
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.create).toHaveBeenCalledWith({
checksum: expect.any(Buffer),
+ checksumAlgorithm: ChecksumAlgorithm.sha1File,
deviceAssetId: 'NONE',
deviceId: 'NONE',
fileCreatedAt: asset.fileCreatedAt,
@@ -705,6 +707,7 @@ describe(MetadataService.name, () => {
expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);
expect(mocks.asset.create).toHaveBeenCalledWith({
checksum: expect.any(Buffer),
+ checksumAlgorithm: ChecksumAlgorithm.sha1File,
deviceAssetId: 'NONE',
deviceId: 'NONE',
fileCreatedAt: asset.fileCreatedAt,
@@ -758,6 +761,7 @@ describe(MetadataService.name, () => {
expect(mocks.storage.readFile).toHaveBeenCalledWith(asset.originalPath, expect.any(Object));
expect(mocks.asset.create).toHaveBeenCalledWith({
checksum: expect.any(Buffer),
+ checksumAlgorithm: ChecksumAlgorithm.sha1File,
deviceAssetId: 'NONE',
deviceId: 'NONE',
fileCreatedAt: asset.fileCreatedAt,
diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts
index bfbcb413c0..c2cf66ad57 100644
--- a/server/src/services/metadata.service.ts
+++ b/server/src/services/metadata.service.ts
@@ -14,6 +14,7 @@ import {
AssetFileType,
AssetType,
AssetVisibility,
+ ChecksumAlgorithm,
DatabaseLock,
ExifOrientation,
ImmichWorker,
@@ -675,6 +676,7 @@ export class MetadataService extends BaseService {
fileModifiedAt: stats.mtime,
localDateTime: dates.localDateTime,
checksum,
+ checksumAlgorithm: ChecksumAlgorithm.sha1File,
ownerId: asset.ownerId,
originalPath: StorageCore.getAndroidMotionPath(asset, motionAssetId),
originalFileName: `${parse(asset.originalFileName).name}.mp4`,
diff --git a/server/test/factories/asset.factory.ts b/server/test/factories/asset.factory.ts
index 321a6f8ddd..9caec31d6f 100644
--- a/server/test/factories/asset.factory.ts
+++ b/server/test/factories/asset.factory.ts
@@ -1,5 +1,5 @@
import { Selectable } from 'kysely';
-import { AssetFileType, AssetStatus, AssetType, AssetVisibility } from 'src/enum';
+import { AssetFileType, AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum';
import { AssetTable } from 'src/schema/tables/asset.table';
import { StackTable } from 'src/schema/tables/stack.table';
import { AssetEditFactory } from 'test/factories/asset-edit.factory';
@@ -53,6 +53,7 @@ export class AssetFactory {
updateId: newUuidV7(),
status: AssetStatus.Active,
checksum: newSha1(),
+ checksumAlgorithm: ChecksumAlgorithm.sha1File,
deviceAssetId: '',
deviceId: '',
duplicateId: null,
diff --git a/server/test/mappers.ts b/server/test/mappers.ts
index 2f3b248576..3d5b34e9c0 100644
--- a/server/test/mappers.ts
+++ b/server/test/mappers.ts
@@ -125,6 +125,7 @@ export const getForMemory = (memory: ReturnType<MemoryFactory['build']>) => ({
export const getForMetadataExtraction = (asset: ReturnType<AssetFactory['build']>) => ({
id: asset.id,
checksum: asset.checksum,
+ checksumAlgorithm: asset.checksumAlgorithm,
deviceAssetId: asset.deviceAssetId,
deviceId: asset.deviceId,
fileCreatedAt: asset.fileCreatedAt,
diff --git a/server/test/medium.factory.ts b/server/test/medium.factory.ts
index a8aa00c2a3..1906fc0ccb 100644
--- a/server/test/medium.factory.ts
+++ b/server/test/medium.factory.ts
@@ -12,6 +12,7 @@ import {
AlbumUserRole,
AssetType,
AssetVisibility,
+ ChecksumAlgorithm,
MemoryType,
SourceType,
SyncEntityType,
@@ -547,6 +548,7 @@ const assetInsert = (asset: Partial<Insertable<AssetTable>> = {}) => {
deviceId: '',
originalFileName: '',
checksum: randomBytes(32),
+ checksumAlgorithm: ChecksumAlgorithm.sha1File,
type: AssetType.Image,
originalPath: '/path/to/something.jpg',
ownerId: 'not-a-valid-uuid',