High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
fix: queue assets missing fullsize files for thumbnail regeneration (#25794)
Min Idzelis092ebe01a550
diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sqlindex 26f04bd9cb..63174f0b0f 100644--- a/server/src/queries/asset.job.repository.sql+++ b/server/src/queries/asset.job.repository.sql@@ -132,6 +132,14 @@ where"assetId" = "asset"."id"and "asset_file"."type" = $3)+ or not exists (+ select+ from+ "asset_file"+ where+ "assetId" = "asset"."id"+ and "asset_file"."type" = $4+ )or "asset"."thumbhash" is null)diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.tsindex 6fec1b38a7..4c6e665c4b 100644--- a/server/src/repositories/asset-job.repository.ts+++ b/server/src/repositories/asset-job.repository.ts@@ -57,8 +57,8 @@ export class AssetJobRepository {.executeTakeFirst();}- @GenerateSql({ params: [false], stream: true })- streamForThumbnailJob(force: boolean) {+ @GenerateSql({ params: [{ force: false, fullsizeEnabled: true }], stream: true })+ streamForThumbnailJob(options: { force: boolean | undefined; fullsizeEnabled: boolean }) {return this.db.selectFrom('asset').select(['asset.id', 'asset.thumbhash'])@@ -66,12 +66,12 @@ export class AssetJobRepository {.select(withEdits).where('asset.deletedAt', 'is', null).where('asset.visibility', '!=', AssetVisibility.Hidden)- .$if(!force, (qb) =>+ .$if(!options.force, (qb) =>qb// If there aren't any entries, metadata extraction hasn't run yet which is required for thumbnails.innerJoin('asset_job_status', 'asset_job_status.assetId', 'asset.id')- .where((eb) =>- eb.or([+ .where((eb) => {+ const conditions = [eb.not((eb) =>eb.exists((qb) =>qb@@ -88,9 +88,25 @@ export class AssetJobRepository {.where('asset_file.type', '=', AssetFileType.Thumbnail),),),- eb('asset.thumbhash', 'is', null),- ]),- ),+ ];++ if (options.fullsizeEnabled) {+ conditions.push(+ eb.not((eb) =>+ eb.exists((qb) =>+ qb+ .selectFrom('asset_file')+ .whereRef('assetId', '=', 'asset.id')+ .where('asset_file.type', '=', AssetFileType.FullSize),+ ),+ ),+ );+ }++ conditions.push(eb('asset.thumbhash', 'is', null));++ return eb.or(conditions);+ }),).stream();}diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.tsindex fa2607faa9..75812e2fcb 100644--- a/server/src/services/media.service.spec.ts+++ b/server/src/services/media.service.spec.ts@@ -23,8 +23,14 @@ import { faceStub } from 'test/fixtures/face.stub';import { probeStub } from 'test/fixtures/media.stub';import { personStub, personThumbnailStub } from 'test/fixtures/person.stub';import { systemConfigStub } from 'test/fixtures/system-config.stub';+import { factory } from 'test/small.factory';import { makeStream, newTestService, ServiceMocks } from 'test/utils';+const filesNoFullsize = [+ factory.assetFile({ type: AssetFileType.Preview }),+ factory.assetFile({ type: AssetFileType.Thumbnail }),+];+const fullsizeBuffer = Buffer.from('embedded image data');const rawBuffer = Buffer.from('raw image data');const extractedBuffer = Buffer.from('embedded image file');@@ -49,7 +55,7 @@ describe(MediaService.name, () => {await sut.handleQueueGenerateThumbnails({ force: true });- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true);+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: true, fullsizeEnabled: false });expect(mocks.job.queueAll).toHaveBeenCalledWith([{name: JobName.AssetGenerateThumbnails,@@ -72,7 +78,7 @@ describe(MediaService.name, () => {await sut.handleQueueGenerateThumbnails({ force: true });- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true);+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: true, fullsizeEnabled: false });expect(mocks.job.queueAll).toHaveBeenCalledWith([{name: JobName.AssetGenerateThumbnails,@@ -87,7 +93,7 @@ describe(MediaService.name, () => {await sut.handleQueueGenerateThumbnails({ force: true });- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true);+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: true, fullsizeEnabled: false });expect(mocks.job.queueAll).toHaveBeenCalledWith([{name: JobName.AssetGenerateThumbnails,@@ -103,7 +109,7 @@ describe(MediaService.name, () => {await sut.handleQueueGenerateThumbnails({ force: false });- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false);+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false });expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });expect(mocks.person.getRandomFace).toHaveBeenCalled();expect(mocks.person.update).toHaveBeenCalledTimes(1);@@ -122,7 +128,7 @@ describe(MediaService.name, () => {mocks.person.getAll.mockReturnValue(makeStream());await sut.handleQueueGenerateThumbnails({ force: false });- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false);+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false });expect(mocks.job.queueAll).toHaveBeenCalledWith([{name: JobName.AssetGenerateThumbnails,@@ -138,7 +144,7 @@ describe(MediaService.name, () => {mocks.person.getAll.mockReturnValue(makeStream());await sut.handleQueueGenerateThumbnails({ force: false });- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false);+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false });expect(mocks.job.queueAll).toHaveBeenCalledWith([{name: JobName.AssetGenerateThumbnails,@@ -154,7 +160,7 @@ describe(MediaService.name, () => {mocks.person.getAll.mockReturnValue(makeStream());await sut.handleQueueGenerateThumbnails({ force: false });- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false);+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false });expect(mocks.job.queueAll).toHaveBeenCalledWith([{name: JobName.AssetGenerateThumbnails,@@ -165,12 +171,43 @@ describe(MediaService.name, () => {expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });});+ it('should queue all assets with missing fullsize when feature is enabled', async () => {+ mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } });+ const asset = { id: factory.uuid(), thumbhash: factory.buffer(), edits: [], files: filesNoFullsize };+ mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([asset]));+ mocks.person.getAll.mockReturnValue(makeStream());+ await sut.handleQueueGenerateThumbnails({ force: false });++ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: true });+ expect(mocks.job.queueAll).toHaveBeenCalledWith([+ {+ name: JobName.AssetGenerateThumbnails,+ data: { id: asset.id },+ },+ ]);++ expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });+ });++ it('should not queue assets with missing fullsize when feature is disabled', async () => {+ mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } });+ const asset = { id: factory.uuid(), thumbhash: factory.buffer(), edits: [], files: filesNoFullsize };+ mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([asset]));+ mocks.person.getAll.mockReturnValue(makeStream());+ await sut.handleQueueGenerateThumbnails({ force: false });++ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false });+ expect(mocks.job.queueAll).toHaveBeenCalledWith([]);++ expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });+ });+it('should queue assets with edits but missing edited thumbnails', async () => {mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.withCropEdit]));mocks.person.getAll.mockReturnValue(makeStream());await sut.handleQueueGenerateThumbnails({ force: false });- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false);+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false });expect(mocks.job.queueAll).toHaveBeenCalledWith([{name: JobName.AssetEditThumbnailGeneration,@@ -181,12 +218,42 @@ describe(MediaService.name, () => {expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });});+ it('should not queue assets with missing edited fullsize when feature is disabled', async () => {+ mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } });+ mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.withCropEdit]));+ mocks.person.getAll.mockReturnValue(makeStream());+ await sut.handleQueueGenerateThumbnails({ force: false });++ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false });+ expect(mocks.job.queueAll).toHaveBeenCalledWith([]);++ expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });+ });++ it('should queue assets with missing fullsize when force is true, regardless of setting', async () => {+ mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } });+ const asset = { id: factory.uuid(), thumbhash: Buffer.from('thumbhash'), edits: [], files: filesNoFullsize };+ mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([asset]));+ mocks.person.getAll.mockReturnValue(makeStream());+ await sut.handleQueueGenerateThumbnails({ force: true });++ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: true, fullsizeEnabled: false });+ expect(mocks.job.queueAll).toHaveBeenCalledWith([+ {+ name: JobName.AssetGenerateThumbnails,+ data: { id: asset.id },+ },+ ]);++ expect(mocks.person.getAll).toHaveBeenCalled();+ });+it('should queue both regular and edited thumbnails for assets with edits when force is true', async () => {mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.withCropEdit]));mocks.person.getAll.mockReturnValue(makeStream());await sut.handleQueueGenerateThumbnails({ force: true });- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true);+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: true, fullsizeEnabled: false });expect(mocks.job.queueAll).toHaveBeenCalledWith([{name: JobName.AssetGenerateThumbnails,diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.tsindex b9b8d74737..00bd0305dd 100644--- a/server/src/services/media.service.ts+++ b/server/src/services/media.service.ts@@ -68,6 +68,7 @@ export class MediaService extends BaseService {@OnJob({ name: JobName.AssetGenerateThumbnailsQueueAll, queue: QueueName.ThumbnailGeneration })async handleQueueGenerateThumbnails({ force }: JobOf<JobName.AssetGenerateThumbnailsQueueAll>): Promise<JobStatus> {+ const config = await this.getConfig({ withCache: true });let jobs: JobItem[] = [];const queueAll = async () => {@@ -75,16 +76,18 @@ export class MediaService extends BaseService {jobs = [];};- for await (const asset of this.assetJobRepository.streamForThumbnailJob(!!force)) {- const assetFiles = getAssetFiles(asset.files);+ const fullsizeEnabled = config.image.fullsize.enabled;+ for await (const asset of this.assetJobRepository.streamForThumbnailJob({ force, fullsizeEnabled })) {+ const { previewFile, thumbnailFile, fullsizeFile, editedPreviewFile, editedThumbnailFile, editedFullsizeFile } =+ getAssetFiles(asset.files);- if (!assetFiles.previewFile || !assetFiles.thumbnailFile || !asset.thumbhash || force) {+ if (force || !previewFile || !thumbnailFile || !asset.thumbhash || (fullsizeEnabled && !fullsizeFile)) {jobs.push({ name: JobName.AssetGenerateThumbnails, data: { id: asset.id } });}if (asset.edits.length > 0 &&- (!assetFiles.editedPreviewFile || !assetFiles.editedThumbnailFile || !assetFiles.editedFullsizeFile || force)+ (force || !editedPreviewFile || !editedThumbnailFile || (fullsizeEnabled && !editedFullsizeFile))) {jobs.push({ name: JobName.AssetEditThumbnailGeneration, data: { id: asset.id } });}diff --git a/server/test/small.factory.ts b/server/test/small.factory.tsindex 6ce4ad1f59..579da1a2d8 100644--- a/server/test/small.factory.ts+++ b/server/test/small.factory.ts@@ -533,6 +533,7 @@ export const factory = {assetEdit: assetEditFactory,tag: tagFactory,uuid: newUuid,+ buffer: () => Buffer.from('this is a fake buffer'),date: newDate,responses: {badRequest: (message: any = null) => ({