Skip to content

High performance self-hosted photo and video management solution.

fix: queue assets missing fullsize files for thumbnail regeneration (#25794)

Min Idzelis092ebe01a550
diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sql
index 26f04bd9cb..63174f0b0f 100644
--- a/server/src/queries/asset.job.repository.sql
+++ b/server/src/queries/asset.job.repository.sql
@@ -132,6 +132,14 @@ where
"assetId" = "asset"."id"
and "asset_file"."type" = $3
)
+ or not exists (
+ select
+ from
+ "asset_file"
+ where
+ "assetId" = "asset"."id"
+ and "asset_file"."type" = $4
+ )
or "asset"."thumbhash" is null
)
diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts
index 6fec1b38a7..4c6e665c4b 100644
--- a/server/src/repositories/asset-job.repository.ts
+++ b/server/src/repositories/asset-job.repository.ts
@@ -57,8 +57,8 @@ export class AssetJobRepository {
.executeTakeFirst();
}
- @GenerateSql({ params: [false], stream: true })
- streamForThumbnailJob(force: boolean) {
+ @GenerateSql({ params: [{ force: false, fullsizeEnabled: true }], stream: true })
+ streamForThumbnailJob(options: { force: boolean | undefined; fullsizeEnabled: boolean }) {
return this.db
.selectFrom('asset')
.select(['asset.id', 'asset.thumbhash'])
@@ -66,12 +66,12 @@ export class AssetJobRepository {
.select(withEdits)
.where('asset.deletedAt', 'is', null)
.where('asset.visibility', '!=', AssetVisibility.Hidden)
- .$if(!force, (qb) =>
+ .$if(!options.force, (qb) =>
qb
// If there aren't any entries, metadata extraction hasn't run yet which is required for thumbnails
.innerJoin('asset_job_status', 'asset_job_status.assetId', 'asset.id')
- .where((eb) =>
- eb.or([
+ .where((eb) => {
+ const conditions = [
eb.not((eb) =>
eb.exists((qb) =>
qb
@@ -88,9 +88,25 @@ export class AssetJobRepository {
.where('asset_file.type', '=', AssetFileType.Thumbnail),
),
),
- eb('asset.thumbhash', 'is', null),
- ]),
- ),
+ ];
+
+ if (options.fullsizeEnabled) {
+ conditions.push(
+ eb.not((eb) =>
+ eb.exists((qb) =>
+ qb
+ .selectFrom('asset_file')
+ .whereRef('assetId', '=', 'asset.id')
+ .where('asset_file.type', '=', AssetFileType.FullSize),
+ ),
+ ),
+ );
+ }
+
+ conditions.push(eb('asset.thumbhash', 'is', null));
+
+ return eb.or(conditions);
+ }),
)
.stream();
}
diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts
index fa2607faa9..75812e2fcb 100644
--- a/server/src/services/media.service.spec.ts
+++ b/server/src/services/media.service.spec.ts
@@ -23,8 +23,14 @@ import { faceStub } from 'test/fixtures/face.stub';
import { probeStub } from 'test/fixtures/media.stub';
import { personStub, personThumbnailStub } from 'test/fixtures/person.stub';
import { systemConfigStub } from 'test/fixtures/system-config.stub';
+import { factory } from 'test/small.factory';
import { makeStream, newTestService, ServiceMocks } from 'test/utils';
+const filesNoFullsize = [
+ factory.assetFile({ type: AssetFileType.Preview }),
+ factory.assetFile({ type: AssetFileType.Thumbnail }),
+];
+
const fullsizeBuffer = Buffer.from('embedded image data');
const rawBuffer = Buffer.from('raw image data');
const extractedBuffer = Buffer.from('embedded image file');
@@ -49,7 +55,7 @@ describe(MediaService.name, () => {
await sut.handleQueueGenerateThumbnails({ force: true });
- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true);
+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: true, fullsizeEnabled: false });
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.AssetGenerateThumbnails,
@@ -72,7 +78,7 @@ describe(MediaService.name, () => {
await sut.handleQueueGenerateThumbnails({ force: true });
- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true);
+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: true, fullsizeEnabled: false });
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.AssetGenerateThumbnails,
@@ -87,7 +93,7 @@ describe(MediaService.name, () => {
await sut.handleQueueGenerateThumbnails({ force: true });
- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true);
+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: true, fullsizeEnabled: false });
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.AssetGenerateThumbnails,
@@ -103,7 +109,7 @@ describe(MediaService.name, () => {
await sut.handleQueueGenerateThumbnails({ force: false });
- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false);
+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false });
expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });
expect(mocks.person.getRandomFace).toHaveBeenCalled();
expect(mocks.person.update).toHaveBeenCalledTimes(1);
@@ -122,7 +128,7 @@ describe(MediaService.name, () => {
mocks.person.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: false });
- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false);
+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false });
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.AssetGenerateThumbnails,
@@ -138,7 +144,7 @@ describe(MediaService.name, () => {
mocks.person.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: false });
- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false);
+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false });
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.AssetGenerateThumbnails,
@@ -154,7 +160,7 @@ describe(MediaService.name, () => {
mocks.person.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: false });
- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false);
+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false });
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.AssetGenerateThumbnails,
@@ -165,12 +171,43 @@ describe(MediaService.name, () => {
expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });
});
+ it('should queue all assets with missing fullsize when feature is enabled', async () => {
+ mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } });
+ const asset = { id: factory.uuid(), thumbhash: factory.buffer(), edits: [], files: filesNoFullsize };
+ mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([asset]));
+ mocks.person.getAll.mockReturnValue(makeStream());
+ await sut.handleQueueGenerateThumbnails({ force: false });
+
+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: true });
+ expect(mocks.job.queueAll).toHaveBeenCalledWith([
+ {
+ name: JobName.AssetGenerateThumbnails,
+ data: { id: asset.id },
+ },
+ ]);
+
+ expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });
+ });
+
+ it('should not queue assets with missing fullsize when feature is disabled', async () => {
+ mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } });
+ const asset = { id: factory.uuid(), thumbhash: factory.buffer(), edits: [], files: filesNoFullsize };
+ mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([asset]));
+ mocks.person.getAll.mockReturnValue(makeStream());
+ await sut.handleQueueGenerateThumbnails({ force: false });
+
+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false });
+ expect(mocks.job.queueAll).toHaveBeenCalledWith([]);
+
+ expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });
+ });
+
it('should queue assets with edits but missing edited thumbnails', async () => {
mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.withCropEdit]));
mocks.person.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: false });
- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false);
+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false });
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.AssetEditThumbnailGeneration,
@@ -181,12 +218,42 @@ describe(MediaService.name, () => {
expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });
});
+ it('should not queue assets with missing edited fullsize when feature is disabled', async () => {
+ mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } });
+ mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.withCropEdit]));
+ mocks.person.getAll.mockReturnValue(makeStream());
+ await sut.handleQueueGenerateThumbnails({ force: false });
+
+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: false, fullsizeEnabled: false });
+ expect(mocks.job.queueAll).toHaveBeenCalledWith([]);
+
+ expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });
+ });
+
+ it('should queue assets with missing fullsize when force is true, regardless of setting', async () => {
+ mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } });
+ const asset = { id: factory.uuid(), thumbhash: Buffer.from('thumbhash'), edits: [], files: filesNoFullsize };
+ mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([asset]));
+ mocks.person.getAll.mockReturnValue(makeStream());
+ await sut.handleQueueGenerateThumbnails({ force: true });
+
+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: true, fullsizeEnabled: false });
+ expect(mocks.job.queueAll).toHaveBeenCalledWith([
+ {
+ name: JobName.AssetGenerateThumbnails,
+ data: { id: asset.id },
+ },
+ ]);
+
+ expect(mocks.person.getAll).toHaveBeenCalled();
+ });
+
it('should queue both regular and edited thumbnails for assets with edits when force is true', async () => {
mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.withCropEdit]));
mocks.person.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: true });
- expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true);
+ expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith({ force: true, fullsizeEnabled: false });
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.AssetGenerateThumbnails,
diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts
index b9b8d74737..00bd0305dd 100644
--- a/server/src/services/media.service.ts
+++ b/server/src/services/media.service.ts
@@ -68,6 +68,7 @@ export class MediaService extends BaseService {
@OnJob({ name: JobName.AssetGenerateThumbnailsQueueAll, queue: QueueName.ThumbnailGeneration })
async handleQueueGenerateThumbnails({ force }: JobOf<JobName.AssetGenerateThumbnailsQueueAll>): Promise<JobStatus> {
+ const config = await this.getConfig({ withCache: true });
let jobs: JobItem[] = [];
const queueAll = async () => {
@@ -75,16 +76,18 @@ export class MediaService extends BaseService {
jobs = [];
};
- for await (const asset of this.assetJobRepository.streamForThumbnailJob(!!force)) {
- const assetFiles = getAssetFiles(asset.files);
+ const fullsizeEnabled = config.image.fullsize.enabled;
+ for await (const asset of this.assetJobRepository.streamForThumbnailJob({ force, fullsizeEnabled })) {
+ const { previewFile, thumbnailFile, fullsizeFile, editedPreviewFile, editedThumbnailFile, editedFullsizeFile } =
+ getAssetFiles(asset.files);
- if (!assetFiles.previewFile || !assetFiles.thumbnailFile || !asset.thumbhash || force) {
+ if (force || !previewFile || !thumbnailFile || !asset.thumbhash || (fullsizeEnabled && !fullsizeFile)) {
jobs.push({ name: JobName.AssetGenerateThumbnails, data: { id: asset.id } });
}
if (
asset.edits.length > 0 &&
- (!assetFiles.editedPreviewFile || !assetFiles.editedThumbnailFile || !assetFiles.editedFullsizeFile || force)
+ (force || !editedPreviewFile || !editedThumbnailFile || (fullsizeEnabled && !editedFullsizeFile))
) {
jobs.push({ name: JobName.AssetEditThumbnailGeneration, data: { id: asset.id } });
}
diff --git a/server/test/small.factory.ts b/server/test/small.factory.ts
index 6ce4ad1f59..579da1a2d8 100644
--- a/server/test/small.factory.ts
+++ b/server/test/small.factory.ts
@@ -533,6 +533,7 @@ export const factory = {
assetEdit: assetEditFactory,
tag: tagFactory,
uuid: newUuid,
+ buffer: () => Buffer.from('this is a fake buffer'),
date: newDate,
responses: {
badRequest: (message: any = null) => ({