High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
fix(server): deleting stacked assets (#25874)
Jason Rasmussen6cdebdd3b3fc
diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sqlindex 50f2c193fc..26f04bd9cb 100644--- a/server/src/queries/asset.job.repository.sql+++ b/server/src/queries/asset.job.repository.sql@@ -430,30 +430,6 @@ select"asset"."originalPath","asset"."isOffline",to_json("asset_exif") as "exifInfo",- (- select- coalesce(json_agg(agg), '[]')- from- (- select- "asset_face".*,- "person" as "person"- from- "asset_face"- left join lateral (- select- "person".*- from- "person"- where- "asset_face"."personId" = "person"."id"- ) as "person" on true- where- "asset_face"."assetId" = "asset"."id"- and "asset_face"."deletedAt" is null- and "asset_face"."isVisible" is true- ) as agg- ) as "faces",(selectcoalesce(json_agg(agg), '[]')@@ -470,27 +446,37 @@ select"asset_file"."assetId" = "asset"."id") as agg) as "files",- to_json("stacked_assets") as "stack"+ to_json("stack_result") as "stack"from"asset"left join "asset_exif" on "asset"."id" = "asset_exif"."assetId"- left join "stack" on "stack"."id" = "asset"."stackId"left join lateral (select"stack"."id","stack"."primaryAssetId",- array_agg("stacked") as "assets"+ (+ select+ coalesce(json_agg(agg), '[]')+ from+ (+ select+ "stack_asset"."id"+ from+ "asset" as "stack_asset"+ where+ "stack_asset"."stackId" = "stack"."id"+ and "stack_asset"."id" != "stack"."primaryAssetId"+ and "stack_asset"."visibility" = $1+ and "stack_asset"."status" != $2+ ) as agg+ ) as "assets"from- "asset" as "stacked"+ "stack"where- "stacked"."deletedAt" is not null- and "stacked"."visibility" = $1- and "stacked"."stackId" = "stack"."id"- group by- "stack"."id"- ) as "stacked_assets" on "stack"."id" is not null+ "stack"."id" = "asset"."stackId"+ ) as "stack_result" on truewhere- "asset"."id" = $2+ "asset"."id" = $3-- AssetJobRepository.streamForVideoConversionselectdiff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.tsindex 1608f7b6f6..6fec1b38a7 100644--- a/server/src/repositories/asset-job.repository.ts+++ b/server/src/repositories/asset-job.repository.ts@@ -1,10 +1,10 @@import { Injectable } from '@nestjs/common';-import { Kysely } from 'kysely';+import { Kysely, sql } from 'kysely';import { jsonArrayFrom } from 'kysely/helpers/postgres';import { InjectKysely } from 'nestjs-kysely';-import { Asset, columns } from 'src/database';+import { columns } from 'src/database';import { DummyValue, GenerateSql } from 'src/decorators';-import { AssetFileType, AssetType, AssetVisibility } from 'src/enum';+import { AssetFileType, AssetStatus, AssetType, AssetVisibility } from 'src/enum';import { DB } from 'src/schema';import {anyUuid,@@ -15,7 +15,6 @@ import {withExif,withExifInner,withFaces,- withFacesAndPeople,withFilePath,withFiles,} from 'src/utils/database';@@ -269,23 +268,29 @@ export class AssetJobRepository {'asset.isOffline',]).$call(withExif)- .select(withFacesAndPeople).select(withFiles)- .leftJoin('stack', 'stack.id', 'asset.stackId').leftJoinLateral((eb) =>eb- .selectFrom('asset as stacked')- .select(['stack.id', 'stack.primaryAssetId'])- .select((eb) => eb.fn<Asset[]>('array_agg', [eb.table('stacked')]).as('assets'))- .where('stacked.deletedAt', 'is not', null)- .where('stacked.visibility', '=', AssetVisibility.Timeline)- .whereRef('stacked.stackId', '=', 'stack.id')- .groupBy('stack.id')- .as('stacked_assets'),- (join) => join.on('stack.id', 'is not', null),+ .selectFrom('stack')+ .whereRef('stack.id', '=', 'asset.stackId')+ .select((eb) => [+ 'stack.id',+ 'stack.primaryAssetId',+ jsonArrayFrom(+ eb+ .selectFrom('asset as stack_asset')+ .select(['stack_asset.id'])+ .whereRef('stack_asset.stackId', '=', 'stack.id')+ .whereRef('stack_asset.id', '!=', 'stack.primaryAssetId')+ .where('stack_asset.visibility', '=', sql.val(AssetVisibility.Timeline))+ .where('stack_asset.status', '!=', sql.val(AssetStatus.Deleted)),+ ).as('assets'),+ ])+ .as('stack_result'),+ (join) => join.onTrue(),)- .select((eb) => toJson(eb, 'stacked_assets').as('stack'))+ .select((eb) => toJson(eb, 'stack_result').as('stack')).where('asset.id', '=', id).executeTakeFirst();}diff --git a/server/src/repositories/storage.repository.ts b/server/src/repositories/storage.repository.tsindex 7345dfef5b..5a1a936e77 100644--- a/server/src/repositories/storage.repository.ts+++ b/server/src/repositories/storage.repository.ts@@ -152,7 +152,7 @@ export class StorageRepository {}async unlinkDir(folder: string, options: { recursive?: boolean; force?: boolean }) {- await fs.rm(folder, options);+ await fs.rm(folder, { ...options, maxRetries: 5, retryDelay: 100 });}async removeEmptyDirs(directory: string, self: boolean = false) {@@ -168,7 +168,13 @@ export class StorageRepository {if (self) {const updated = await fs.readdir(directory);if (updated.length === 0) {- await fs.rmdir(directory);+ try {+ await fs.rmdir(directory);+ } catch (error: Error | any) {+ if (error.code !== 'ENOTEMPTY') {+ this.logger.warn(`Attempted to remove directory, but failed: ${error}`);+ }+ }}}}diff --git a/server/src/services/asset.service.spec.ts b/server/src/services/asset.service.spec.tsindex eca49bc14e..707faa326d 100755--- a/server/src/services/asset.service.spec.ts+++ b/server/src/services/asset.service.spec.ts@@ -8,7 +8,6 @@ import { AssetStats } from 'src/repositories/asset.repository';import { AssetService } from 'src/services/asset.service';import { assetStub } from 'test/fixtures/asset.stub';import { authStub } from 'test/fixtures/auth.stub';-import { faceStub } from 'test/fixtures/face.stub';import { userStub } from 'test/fixtures/user.stub';import { factory } from 'test/small.factory';import { makeStream, newTestService, ServiceMocks } from 'test/utils';@@ -565,12 +564,11 @@ describe(AssetService.name, () => {});describe('handleAssetDeletion', () => {- it('should remove faces', async () => {- const assetWithFace = { ...assetStub.image, faces: [faceStub.face1, faceStub.mergeFace1] };+ it('should clean up files', async () => {+ const asset = assetStub.image;+ mocks.assetJob.getForAssetDeletion.mockResolvedValue(asset);- mocks.assetJob.getForAssetDeletion.mockResolvedValue(assetWithFace);-- await sut.handleAssetDeletion({ id: assetWithFace.id, deleteOnDisk: true });+ await sut.handleAssetDeletion({ id: asset.id, deleteOnDisk: true });expect(mocks.job.queue.mock.calls).toEqual([[@@ -581,38 +579,29 @@ describe(AssetService.name, () => {'/uploads/user-id/webp/path.ext','/uploads/user-id/thumbs/path.jpg','/uploads/user-id/fullsize/path.webp',- assetWithFace.originalPath,+ asset.originalPath,],},},],]);-- expect(mocks.asset.remove).toHaveBeenCalledWith(assetWithFace);- });-- it('should update stack primary asset if deleted asset was primary asset in a stack', async () => {- mocks.stack.update.mockResolvedValue(factory.stack() as any);- mocks.assetJob.getForAssetDeletion.mockResolvedValue(assetStub.primaryImage);-- await sut.handleAssetDeletion({ id: assetStub.primaryImage.id, deleteOnDisk: true });-- expect(mocks.stack.update).toHaveBeenCalledWith('stack-1', {- id: 'stack-1',- primaryAssetId: 'stack-child-asset-1',- });+ expect(mocks.asset.remove).toHaveBeenCalledWith(asset);});it('should delete the entire stack if deleted asset was the primary asset and the stack would only contain one asset afterwards', async () => {mocks.stack.delete.mockResolvedValue();mocks.assetJob.getForAssetDeletion.mockResolvedValue({...assetStub.primaryImage,- stack: { ...assetStub.primaryImage.stack, assets: assetStub.primaryImage.stack!.assets.slice(0, 2) },+ stack: {+ id: 'stack-id',+ primaryAssetId: assetStub.primaryImage.id,+ assets: [{ id: 'one-asset' }],+ },});await sut.handleAssetDeletion({ id: assetStub.primaryImage.id, deleteOnDisk: true });- expect(mocks.stack.delete).toHaveBeenCalledWith('stack-1');+ expect(mocks.stack.delete).toHaveBeenCalledWith('stack-id');});it('should delete a live photo', async () => {diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.tsindex 066084ed45..23ba29e7b8 100644--- a/server/src/services/asset.service.ts+++ b/server/src/services/asset.service.ts@@ -327,10 +327,11 @@ export class AssetService extends BaseService {return JobStatus.Failed;}- // Replace the parent of the stack children with a new asset+ // replace the parent of the stack children with a new assetif (asset.stack?.primaryAssetId === id) {- const stackAssetIds = asset.stack?.assets.map((a) => a.id) ?? [];- if (stackAssetIds.length > 2) {+ // this only includes timeline visible assets and excludes the primary asset+ const stackAssetIds = asset.stack.assets.map((a) => a.id);+ if (stackAssetIds.length >= 2) {const newPrimaryAssetId = stackAssetIds.find((a) => a !== id)!;await this.stackRepository.update(asset.stack.id, {id: asset.stack.id,diff --git a/server/test/medium/specs/services/asset.service.spec.ts b/server/test/medium/specs/services/asset.service.spec.tsindex d0949c153c..1e34544e38 100644--- a/server/test/medium/specs/services/asset.service.spec.ts+++ b/server/test/medium/specs/services/asset.service.spec.ts@@ -1,5 +1,5 @@import { Kysely } from 'kysely';-import { AssetFileType, AssetMetadataKey, JobName, SharedLinkType } from 'src/enum';+import { AssetFileType, AssetMetadataKey, AssetStatus, JobName, SharedLinkType } from 'src/enum';import { AccessRepository } from 'src/repositories/access.repository';import { AlbumRepository } from 'src/repositories/album.repository';import { AssetJobRepository } from 'src/repositories/asset-job.repository';@@ -246,6 +246,66 @@ describe(AssetService.name, () => {});});+ it('should delete a stacked primary asset (2 assets)', async () => {+ const { sut, ctx } = setup();+ ctx.getMock(EventRepository).emit.mockResolvedValue();+ ctx.getMock(JobRepository).queue.mockResolvedValue();+ const { user } = await ctx.newUser();+ const { asset: asset1 } = await ctx.newAsset({ ownerId: user.id });+ const { asset: asset2 } = await ctx.newAsset({ ownerId: user.id });+ const { stack, result } = await ctx.newStack({ ownerId: user.id }, [asset1.id, asset2.id]);++ const stackRepo = ctx.get(StackRepository);++ expect(result).toMatchObject({ primaryAssetId: asset1.id });++ await sut.handleAssetDeletion({ id: asset1.id, deleteOnDisk: true });++ // stack is deleted as well+ await expect(stackRepo.getById(stack.id)).resolves.toBe(undefined);+ });++ it('should delete a stacked primary asset (3 assets)', async () => {+ const { sut, ctx } = setup();+ ctx.getMock(EventRepository).emit.mockResolvedValue();+ ctx.getMock(JobRepository).queue.mockResolvedValue();+ const { user } = await ctx.newUser();+ const { asset: asset1 } = await ctx.newAsset({ ownerId: user.id });+ const { asset: asset2 } = await ctx.newAsset({ ownerId: user.id });+ const { asset: asset3 } = await ctx.newAsset({ ownerId: user.id });+ const { stack, result } = await ctx.newStack({ ownerId: user.id }, [asset1.id, asset2.id, asset3.id]);++ expect(result).toMatchObject({ primaryAssetId: asset1.id });++ await sut.handleAssetDeletion({ id: asset1.id, deleteOnDisk: true });++ // new primary asset is picked+ await expect(ctx.get(StackRepository).getById(stack.id)).resolves.toMatchObject({ primaryAssetId: asset2.id });+ });++ it('should delete a stacked primary asset (3 trashed assets)', async () => {+ const { sut, ctx } = setup();+ ctx.getMock(EventRepository).emit.mockResolvedValue();+ ctx.getMock(JobRepository).queue.mockResolvedValue();+ const { user } = await ctx.newUser();+ const { asset: asset1 } = await ctx.newAsset({ ownerId: user.id });+ const { asset: asset2 } = await ctx.newAsset({ ownerId: user.id });+ const { asset: asset3 } = await ctx.newAsset({ ownerId: user.id });+ const { stack, result } = await ctx.newStack({ ownerId: user.id }, [asset1.id, asset2.id, asset3.id]);++ await ctx.get(AssetRepository).updateAll([asset1.id, asset2.id, asset3.id], {+ deletedAt: new Date(),+ status: AssetStatus.Deleted,+ });++ expect(result).toMatchObject({ primaryAssetId: asset1.id });++ await sut.handleAssetDeletion({ id: asset1.id, deleteOnDisk: true });++ // stack is deleted as well+ await expect(ctx.get(StackRepository).getById(stack.id)).resolves.toBe(undefined);+ });+it('should not delete offline assets', async () => {const { sut, ctx } = setup();ctx.getMock(EventRepository).emit.mockResolvedValue();