Skip to content

High performance self-hosted photo and video management solution.

refactor: queue asset deletes via stream (#16706)

Jason Rasmussen3f06a494a905
diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts
index 1efcb3875b..5f020ba836 100644
--- a/server/src/repositories/asset.repository.ts
+++ b/server/src/repositories/asset.repository.ts
@@ -637,6 +637,14 @@ export class AssetRepository {
return this.storageTemplateAssetQuery().stream() as AsyncIterableIterator<StorageAsset>;
}
+ streamDeletedAssets(trashedBefore: Date) {
+ return this.db
+ .selectFrom('assets')
+ .select(['id', 'isOffline'])
+ .where('assets.deletedAt', '<=', trashedBefore)
+ .stream();
+ }
+
@GenerateSql(
...Object.values(WithProperty).map((property) => ({
name: property,
diff --git a/server/src/services/asset.service.spec.ts b/server/src/services/asset.service.spec.ts
index f91f600bb1..9286008581 100755
--- a/server/src/services/asset.service.spec.ts
+++ b/server/src/services/asset.service.spec.ts
@@ -11,7 +11,8 @@ import { authStub } from 'test/fixtures/auth.stub';
import { faceStub } from 'test/fixtures/face.stub';
import { partnerStub } from 'test/fixtures/partner.stub';
import { userStub } from 'test/fixtures/user.stub';
-import { newTestService, ServiceMocks } from 'test/utils';
+import { factory } from 'test/small.factory';
+import { makeStream, newTestService, ServiceMocks } from 'test/utils';
import { vitest } from 'vitest';
const stats: AssetStats = {
@@ -473,28 +474,30 @@ describe(AssetService.name, () => {
});
it('should immediately queue assets for deletion if trash is disabled', async () => {
- mocks.asset.getAll.mockResolvedValue({ hasNextPage: false, items: [assetStub.image] });
+ const asset = factory.asset({ isOffline: false });
+
+ mocks.asset.streamDeletedAssets.mockReturnValue(makeStream([asset]));
mocks.systemMetadata.get.mockResolvedValue({ trash: { enabled: false } });
await expect(sut.handleAssetDeletionCheck()).resolves.toBe(JobStatus.SUCCESS);
- expect(mocks.asset.getAll).toHaveBeenCalledWith(expect.anything(), { trashedBefore: new Date() });
+ expect(mocks.asset.streamDeletedAssets).toHaveBeenCalledWith(new Date());
expect(mocks.job.queueAll).toHaveBeenCalledWith([
- { name: JobName.ASSET_DELETION, data: { id: assetStub.image.id, deleteOnDisk: true } },
+ { name: JobName.ASSET_DELETION, data: { id: asset.id, deleteOnDisk: true } },
]);
});
it('should queue assets for deletion after trash duration', async () => {
- mocks.asset.getAll.mockResolvedValue({ hasNextPage: false, items: [assetStub.image] });
+ const asset = factory.asset({ isOffline: false });
+
+ mocks.asset.streamDeletedAssets.mockReturnValue(makeStream([asset]));
mocks.systemMetadata.get.mockResolvedValue({ trash: { enabled: true, days: 7 } });
await expect(sut.handleAssetDeletionCheck()).resolves.toBe(JobStatus.SUCCESS);
- expect(mocks.asset.getAll).toHaveBeenCalledWith(expect.anything(), {
- trashedBefore: DateTime.now().minus({ days: 7 }).toJSDate(),
- });
+ expect(mocks.asset.streamDeletedAssets).toHaveBeenCalledWith(DateTime.now().minus({ days: 7 }).toJSDate());
expect(mocks.job.queueAll).toHaveBeenCalledWith([
- { name: JobName.ASSET_DELETION, data: { id: assetStub.image.id, deleteOnDisk: true } },
+ { name: JobName.ASSET_DELETION, data: { id: asset.id, deleteOnDisk: true } },
]);
});
});
diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.ts
index df66d405b7..56b7f7743c 100644
--- a/server/src/services/asset.service.ts
+++ b/server/src/services/asset.service.ts
@@ -25,7 +25,6 @@ import { AssetStatus, JobName, JobStatus, Permission, QueueName } from 'src/enum
import { BaseService } from 'src/services/base.service';
import { ISidecarWriteJob, JobItem, JobOf } from 'src/types';
import { getAssetFiles, getMyPartnerIds, onAfterUnlink, onBeforeLink, onBeforeUnlink } from 'src/utils/asset.util';
-import { usePagination } from 'src/utils/pagination';
@Injectable()
export class AssetService extends BaseService {
@@ -156,22 +155,30 @@ export class AssetService extends BaseService {
const trashedBefore = DateTime.now()
.minus(Duration.fromObject({ days: trashedDays }))
.toJSDate();
- const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
- this.assetRepository.getAll(pagination, { trashedBefore }),
- );
- for await (const assets of assetPagination) {
- await this.jobRepository.queueAll(
- assets.map((asset) => ({
- name: JobName.ASSET_DELETION,
- data: {
- id: asset.id,
- deleteOnDisk: !asset.isOffline,
- },
- })),
- );
+ let chunk: Array<{ id: string; isOffline: boolean }> = [];
+ const queueChunk = async () => {
+ if (chunk.length > 0) {
+ await this.jobRepository.queueAll(
+ chunk.map(({ id, isOffline }) => ({
+ name: JobName.ASSET_DELETION,
+ data: { id, deleteOnDisk: !isOffline },
+ })),
+ );
+ chunk = [];
+ }
+ };
+
+ const assets = this.assetRepository.streamDeletedAssets(trashedBefore);
+ for await (const asset of assets) {
+ chunk.push(asset);
+ if (chunk.length >= JOBS_ASSET_PAGINATION_SIZE) {
+ await queueChunk();
+ }
}
+ await queueChunk();
+
return JobStatus.SUCCESS;
}
diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.ts
index 66b56b0ecc..19464f7ff2 100644
--- a/server/test/repositories/asset.repository.mock.ts
+++ b/server/test/repositories/asset.repository.mock.ts
@@ -46,5 +46,6 @@ export const newAssetRepositoryMock = (): Mocked<RepositoryInterface<AssetReposi
updateByLibraryId: vitest.fn(),
streamStorageTemplateAssets: vitest.fn(),
getStorageTemplateAsset: vitest.fn(),
+ streamDeletedAssets: vitest.fn(),
};
};