High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
fix: motion video extraction race condition (#21285)
Jason Rasmussen868d5f56e258
diff --git a/server/src/services/asset-media.service.ts b/server/src/services/asset-media.service.tsindex 69e1dfd3a0..54bbedea9c 100644--- a/server/src/services/asset-media.service.ts+++ b/server/src/services/asset-media.service.ts@@ -27,7 +27,7 @@ import { BaseService } from 'src/services/base.service';import { UploadFile } from 'src/types';import { requireUploadAccess } from 'src/utils/access';import { asRequest, getAssetFiles, onBeforeLink } from 'src/utils/asset.util';-import { ASSET_CHECKSUM_CONSTRAINT } from 'src/utils/database';+import { isAssetChecksumConstraint } from 'src/utils/database';import { getFilenameExtension, getFileNameWithoutExtension, ImmichFileResponse } from 'src/utils/file';import { mimeTypes } from 'src/utils/mime-types';import { fromChecksum } from 'src/utils/request';@@ -318,7 +318,7 @@ export class AssetMediaService extends BaseService {});// handle duplicates with a success response- if (error.constraint_name === ASSET_CHECKSUM_CONSTRAINT) {+ if (isAssetChecksumConstraint(error)) {const duplicateId = await this.assetRepository.getUploadAssetIdByChecksum(auth.user.id, file.checksum);if (!duplicateId) {this.logger.error(`Error locating duplicate for checksum constraint`);diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.tsindex c675b7200d..ac2b927510 100644--- a/server/src/services/metadata.service.ts+++ b/server/src/services/metadata.service.ts@@ -29,6 +29,7 @@ import { AssetFaceTable } from 'src/schema/tables/asset-face.table';import { PersonTable } from 'src/schema/tables/person.table';import { BaseService } from 'src/services/base.service';import { JobItem, JobOf } from 'src/types';+import { isAssetChecksumConstraint } from 'src/utils/database';import { isFaceImportEnabled } from 'src/utils/misc';import { upsertTags } from 'src/utils/tag';@@ -545,47 +546,62 @@ export class MetadataService extends BaseService {});}const checksum = this.cryptoRepository.hashSha1(video);+ const checksumQuery = { ownerId: asset.ownerId, libraryId: asset.libraryId ?? undefined, checksum };++ let motionAsset = await this.assetRepository.getByChecksum(checksumQuery);+ let isNewMotionAsset = false;++ if (!motionAsset) {+ try {+ const motionAssetId = this.cryptoRepository.randomUUID();+ motionAsset = await this.assetRepository.create({+ id: motionAssetId,+ libraryId: asset.libraryId,+ type: AssetType.Video,+ fileCreatedAt: dates.dateTimeOriginal,+ fileModifiedAt: stats.mtime,+ localDateTime: dates.localDateTime,+ checksum,+ ownerId: asset.ownerId,+ originalPath: StorageCore.getAndroidMotionPath(asset, motionAssetId),+ originalFileName: `${path.parse(asset.originalFileName).name}.mp4`,+ visibility: AssetVisibility.Hidden,+ deviceAssetId: 'NONE',+ deviceId: 'NONE',+ });++ isNewMotionAsset = true;++ if (!asset.isExternal) {+ await this.userRepository.updateUsage(asset.ownerId, video.byteLength);+ }+ } catch (error) {+ if (!isAssetChecksumConstraint(error)) {+ throw error;+ }++ motionAsset = await this.assetRepository.getByChecksum(checksumQuery);+ if (!motionAsset) {+ this.logger.warn(`Unable to find existing motion video asset for ${asset.id}: ${asset.originalPath}`);+ return;+ }+ }+ }- let motionAsset = await this.assetRepository.getByChecksum({- ownerId: asset.ownerId,- libraryId: asset.libraryId ?? undefined,- checksum,- });- if (motionAsset) {+ if (!isNewMotionAsset) {this.logger.debugFn(() => {const base64Checksum = checksum.toString('base64');return `Motion asset with checksum ${base64Checksum} already exists for asset ${asset.id}: ${asset.originalPath}`;});+ }- // Hide the motion photo video asset if it's not already hidden to prepare for linking- if (motionAsset.visibility === AssetVisibility.Timeline) {- await this.assetRepository.update({- id: motionAsset.id,- visibility: AssetVisibility.Hidden,- });- this.logger.log(`Hid unlinked motion photo video asset (${motionAsset.id})`);- }- } else {- const motionAssetId = this.cryptoRepository.randomUUID();- motionAsset = await this.assetRepository.create({- id: motionAssetId,- libraryId: asset.libraryId,- type: AssetType.Video,- fileCreatedAt: dates.dateTimeOriginal,- fileModifiedAt: stats.mtime,- localDateTime: dates.localDateTime,- checksum,- ownerId: asset.ownerId,- originalPath: StorageCore.getAndroidMotionPath(asset, motionAssetId),- originalFileName: `${path.parse(asset.originalFileName).name}.mp4`,+ // Hide the motion photo video asset if it's not already hidden to prepare for linking+ if (motionAsset.visibility === AssetVisibility.Timeline) {+ await this.assetRepository.update({+ id: motionAsset.id,visibility: AssetVisibility.Hidden,- deviceAssetId: 'NONE',- deviceId: 'NONE',});-- if (!asset.isExternal) {- await this.userRepository.updateUsage(asset.ownerId, video.byteLength);- }+ this.logger.log(`Hid unlinked motion photo video asset (${motionAsset.id})`);}if (asset.livePhotoVideoId !== motionAsset.id) {diff --git a/server/src/utils/database.ts b/server/src/utils/database.tsindex 1ef9b8e926..d9fe6b7897 100644--- a/server/src/utils/database.ts+++ b/server/src/utils/database.ts@@ -14,7 +14,7 @@ import {import { PostgresJSDialect } from 'kysely-postgres-js';import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';import { parse } from 'pg-connection-string';-import postgres, { Notice } from 'postgres';+import postgres, { Notice, PostgresError } from 'postgres';import { columns, Exif, Person } from 'src/database';import { AssetFileType, AssetVisibility, DatabaseExtension, DatabaseSslMode } from 'src/enum';import { AssetSearchBuilderOptions } from 'src/repositories/search.repository';@@ -153,6 +153,10 @@ export function toJson<DB, TB extends keyof DB & string, T extends TB | Expressiexport const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_checksum';+export const isAssetChecksumConstraint = (error: unknown) => {+ return (error as PostgresError)?.constraint_name === 'UQ_assets_owner_checksum';+};+export function withDefaultVisibility<O>(qb: SelectQueryBuilder<DB, 'asset', O>) {return qb.where('asset.visibility', 'in', [sql.lit(AssetVisibility.Archive), sql.lit(AssetVisibility.Timeline)]);}