High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat(server): add checksum algorithm field (#26573)
Jonathan Jogenfors22bf7c200535
diff --git a/server/src/database.ts b/server/src/database.tsindex f4878b2cee..4f339624e6 100644--- a/server/src/database.ts+++ b/server/src/database.ts@@ -5,6 +5,7 @@ import {AssetFileType,AssetType,AssetVisibility,+ ChecksumAlgorithm,MemoryType,Permission,PluginContext,@@ -112,6 +113,7 @@ export type Memory = {export type Asset = {id: string;checksum: Buffer<ArrayBufferLike>;+ checksumAlgorithm: ChecksumAlgorithm;deviceAssetId: string;deviceId: string;fileCreatedAt: Date;@@ -330,6 +332,7 @@ export const columns = {asset: ['asset.id','asset.checksum',+ 'asset.checksumAlgorithm','asset.deviceAssetId','asset.deviceId','asset.fileCreatedAt',diff --git a/server/src/dtos/asset-response.dto.ts b/server/src/dtos/asset-response.dto.tsindex 8b38b2e124..2c2f57bbb2 100644--- a/server/src/dtos/asset-response.dto.ts+++ b/server/src/dtos/asset-response.dto.ts@@ -13,7 +13,7 @@ import {} from 'src/dtos/person.dto';import { TagResponseDto, mapTag } from 'src/dtos/tag.dto';import { UserResponseDto, mapUser } from 'src/dtos/user.dto';-import { AssetStatus, AssetType, AssetVisibility } from 'src/enum';+import { AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum';import { ImageDimensions, MaybeDehydrated } from 'src/types';import { getDimensions } from 'src/utils/asset.util';import { hexOrBufferToBase64 } from 'src/utils/bytes';@@ -148,6 +148,7 @@ export type MapAsset = {updateId: string;status: AssetStatus;checksum: Buffer<ArrayBufferLike>;+ checksumAlgorithm: ChecksumAlgorithm;deviceAssetId: string;deviceId: string;duplicateId: string | null;diff --git a/server/src/enum.ts b/server/src/enum.tsindex 60f45efd6e..1277a39036 100644--- a/server/src/enum.ts+++ b/server/src/enum.ts@@ -37,6 +37,11 @@ export enum AssetType {Other = 'OTHER',}+export enum ChecksumAlgorithm {+ sha1File = 'sha1', // sha1 checksum of the whole file contents+ sha1Path = 'sha1-path', // sha1 checksum of "path:" plus the file path, currently used in external libraries, deprecated+}+export enum AssetFileType {/*** An full/large-size image extracted/converted from RAW photosdiff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sqlindex cf5b8f02dc..554aa3b195 100644--- a/server/src/queries/asset.job.repository.sql+++ b/server/src/queries/asset.job.repository.sql@@ -249,6 +249,7 @@ whereselect"asset"."id","asset"."checksum",+ "asset"."checksumAlgorithm","asset"."deviceAssetId","asset"."deviceId","asset"."fileCreatedAt",diff --git a/server/src/schema/enums.ts b/server/src/schema/enums.tsindex c68f152779..f63a09c462 100644--- a/server/src/schema/enums.ts+++ b/server/src/schema/enums.ts@@ -1,5 +1,5 @@import { registerEnum } from '@immich/sql-tools';-import { AssetStatus, AssetVisibility, SourceType } from 'src/enum';+import { AssetStatus, AssetVisibility, ChecksumAlgorithm, SourceType } from 'src/enum';export const assets_status_enum = registerEnum({name: 'assets_status_enum',@@ -15,3 +15,8 @@ export const asset_visibility_enum = registerEnum({name: 'asset_visibility_enum',values: Object.values(AssetVisibility),});++export const asset_checksum_algorithm_enum = registerEnum({+ name: 'asset_checksum_algorithm_enum',+ values: Object.values(ChecksumAlgorithm),+});diff --git a/server/src/schema/migrations/1774548649115-AddChecksumAlgorithm.ts.ts b/server/src/schema/migrations/1774548649115-AddChecksumAlgorithm.ts.tsnew file mode 100644index 0000000000..477e7d0e48--- /dev/null+++ b/server/src/schema/migrations/1774548649115-AddChecksumAlgorithm.ts.ts@@ -0,0 +1,21 @@+import { Kysely, sql } from 'kysely';++export async function up(db: Kysely<any>): Promise<void> {+ await sql`CREATE TYPE "asset_checksum_algorithm_enum" AS ENUM ('sha1','sha1-path');`.execute(db);+ await sql`ALTER TABLE "asset" ADD "checksumAlgorithm" asset_checksum_algorithm_enum;`.execute(db);++ await sql`+ UPDATE "asset"+ SET "checksumAlgorithm" = CASE+ WHEN "isExternal" = true THEN 'sha1-path'::asset_checksum_algorithm_enum+ ELSE 'sha1'::asset_checksum_algorithm_enum+ END+ `.execute(db);++ await sql`ALTER TABLE "asset" ALTER COLUMN "checksumAlgorithm" SET NOT NULL;`.execute(db);+}++export async function down(db: Kysely<any>): Promise<void> {+ await sql`ALTER TABLE "asset" DROP COLUMN "checksumAlgorithm";`.execute(db);+ await sql`DROP TYPE "asset_checksum_algorithm_enum";`.execute(db);+}diff --git a/server/src/schema/tables/asset.table.ts b/server/src/schema/tables/asset.table.tsindex 8bdaa59bc6..7418dab102 100644--- a/server/src/schema/tables/asset.table.ts+++ b/server/src/schema/tables/asset.table.ts@@ -12,8 +12,8 @@ import {UpdateDateColumn,} from '@immich/sql-tools';import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators';-import { AssetStatus, AssetType, AssetVisibility } from 'src/enum';-import { asset_visibility_enum, assets_status_enum } from 'src/schema/enums';+import { AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum';+import { asset_checksum_algorithm_enum, asset_visibility_enum, assets_status_enum } from 'src/schema/enums';import { asset_delete_audit } from 'src/schema/functions';import { LibraryTable } from 'src/schema/tables/library.table';import { StackTable } from 'src/schema/tables/stack.table';@@ -95,6 +95,9 @@ export class AssetTable {@Column({ type: 'bytea', index: true })checksum!: Buffer; // sha1 checksum+ @Column({ enum: asset_checksum_algorithm_enum })+ checksumAlgorithm!: ChecksumAlgorithm;+@ForeignKeyColumn(() => AssetTable, { nullable: true, onUpdate: 'CASCADE', onDelete: 'SET NULL' })livePhotoVideoId!: string | null;diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.tsindex 03677054ec..8b8efd19c5 100644--- a/server/src/services/asset-media.service.ts+++ b/server/src/services/asset-media.service.ts@@ -27,6 +27,7 @@ import {AssetStatus,AssetVisibility,CacheControl,+ ChecksumAlgorithm,JobName,Permission,StorageFolder,@@ -425,6 +426,7 @@ export class AssetMediaService extends BaseService {deviceId: asset.deviceId,type: asset.type,checksum: asset.checksum,+ checksumAlgorithm: asset.checksumAlgorithm,fileCreatedAt: asset.fileCreatedAt,localDateTime: asset.localDateTime,fileModifiedAt: asset.fileModifiedAt,@@ -446,6 +448,7 @@ export class AssetMediaService extends BaseService {libraryId: null,checksum: file.checksum,+ checksumAlgorithm: ChecksumAlgorithm.sha1File,originalPath: file.originalPath,deviceAssetId: dto.deviceAssetId,diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.tsindex 841fa4743c..9f2d69bab5 100644--- a/server/src/services/library.service.ts+++ b/server/src/services/library.service.ts@@ -17,7 +17,17 @@ import {ValidateLibraryImportPathResponseDto,ValidateLibraryResponseDto,} from 'src/dtos/library.dto';-import { AssetStatus, AssetType, CronJob, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum';+import {+ AssetStatus,+ AssetType,+ ChecksumAlgorithm,+ CronJob,+ DatabaseLock,+ ImmichWorker,+ JobName,+ JobStatus,+ QueueName,+} from 'src/enum';import { ArgOf } from 'src/repositories/event.repository';import { AssetSyncResult } from 'src/repositories/library.repository';import { AssetTable } from 'src/schema/tables/asset.table';@@ -400,6 +410,7 @@ export class LibraryService extends BaseService {ownerId,libraryId,checksum: this.cryptoRepository.hashSha1(`path:${assetPath}`),+ checksumAlgorithm: ChecksumAlgorithm.sha1Path,originalPath: assetPath,fileCreatedAt: stat.mtime,diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.tsindex cb35e21d0a..f9d17079e3 100644--- a/server/src/services/metadata.service.spec.ts+++ b/server/src/services/metadata.service.spec.ts@@ -7,6 +7,7 @@ import {AssetFileType,AssetType,AssetVisibility,+ ChecksumAlgorithm,ExifOrientation,ImmichWorker,JobName,@@ -652,6 +653,7 @@ describe(MetadataService.name, () => {expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);expect(mocks.asset.create).toHaveBeenCalledWith({checksum: expect.any(Buffer),+ checksumAlgorithm: ChecksumAlgorithm.sha1File,deviceAssetId: 'NONE',deviceId: 'NONE',fileCreatedAt: asset.fileCreatedAt,@@ -705,6 +707,7 @@ describe(MetadataService.name, () => {expect(mocks.assetJob.getForMetadataExtraction).toHaveBeenCalledWith(asset.id);expect(mocks.asset.create).toHaveBeenCalledWith({checksum: expect.any(Buffer),+ checksumAlgorithm: ChecksumAlgorithm.sha1File,deviceAssetId: 'NONE',deviceId: 'NONE',fileCreatedAt: asset.fileCreatedAt,@@ -758,6 +761,7 @@ describe(MetadataService.name, () => {expect(mocks.storage.readFile).toHaveBeenCalledWith(asset.originalPath, expect.any(Object));expect(mocks.asset.create).toHaveBeenCalledWith({checksum: expect.any(Buffer),+ checksumAlgorithm: ChecksumAlgorithm.sha1File,deviceAssetId: 'NONE',deviceId: 'NONE',fileCreatedAt: asset.fileCreatedAt,diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.tsindex bfbcb413c0..c2cf66ad57 100644--- a/server/src/services/metadata.service.ts+++ b/server/src/services/metadata.service.ts@@ -14,6 +14,7 @@ import {AssetFileType,AssetType,AssetVisibility,+ ChecksumAlgorithm,DatabaseLock,ExifOrientation,ImmichWorker,@@ -675,6 +676,7 @@ export class MetadataService extends BaseService {fileModifiedAt: stats.mtime,localDateTime: dates.localDateTime,checksum,+ checksumAlgorithm: ChecksumAlgorithm.sha1File,ownerId: asset.ownerId,originalPath: StorageCore.getAndroidMotionPath(asset, motionAssetId),originalFileName: `${parse(asset.originalFileName).name}.mp4`,diff --git a/server/test/factories/asset.factory.ts b/server/test/factories/asset.factory.tsindex 321a6f8ddd..9caec31d6f 100644--- a/server/test/factories/asset.factory.ts+++ b/server/test/factories/asset.factory.ts@@ -1,5 +1,5 @@import { Selectable } from 'kysely';-import { AssetFileType, AssetStatus, AssetType, AssetVisibility } from 'src/enum';+import { AssetFileType, AssetStatus, AssetType, AssetVisibility, ChecksumAlgorithm } from 'src/enum';import { AssetTable } from 'src/schema/tables/asset.table';import { StackTable } from 'src/schema/tables/stack.table';import { AssetEditFactory } from 'test/factories/asset-edit.factory';@@ -53,6 +53,7 @@ export class AssetFactory {updateId: newUuidV7(),status: AssetStatus.Active,checksum: newSha1(),+ checksumAlgorithm: ChecksumAlgorithm.sha1File,deviceAssetId: '',deviceId: '',duplicateId: null,diff --git a/server/test/mappers.ts b/server/test/mappers.tsindex 2f3b248576..3d5b34e9c0 100644--- a/server/test/mappers.ts+++ b/server/test/mappers.ts@@ -125,6 +125,7 @@ export const getForMemory = (memory: ReturnType<MemoryFactory['build']>) => ({export const getForMetadataExtraction = (asset: ReturnType<AssetFactory['build']>) => ({id: asset.id,checksum: asset.checksum,+ checksumAlgorithm: asset.checksumAlgorithm,deviceAssetId: asset.deviceAssetId,deviceId: asset.deviceId,fileCreatedAt: asset.fileCreatedAt,diff --git a/server/test/medium.factory.ts b/server/test/medium.factory.tsindex a8aa00c2a3..1906fc0ccb 100644--- a/server/test/medium.factory.ts+++ b/server/test/medium.factory.ts@@ -12,6 +12,7 @@ import {AlbumUserRole,AssetType,AssetVisibility,+ ChecksumAlgorithm,MemoryType,SourceType,SyncEntityType,@@ -547,6 +548,7 @@ const assetInsert = (asset: Partial<Insertable<AssetTable>> = {}) => {deviceId: '',originalFileName: '',checksum: randomBytes(32),+ checksumAlgorithm: ChecksumAlgorithm.sha1File,type: AssetType.Image,originalPath: '/path/to/something.jpg',ownerId: 'not-a-valid-uuid',