High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
refactor: stream for sidecar (#17995)
Alex732b06eec8fc
diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sqlindex ecf8f0b475..a56b1c61a4 100644--- a/server/src/queries/asset.job.repository.sql+++ b/server/src/queries/asset.job.repository.sql@@ -470,6 +470,18 @@ fromwhere"assets"."deletedAt" <= $1+-- AssetJobRepository.streamForSidecar+select+ "assets"."id"+from+ "assets"+where+ (+ "assets"."sidecarPath" = $1+ or "assets"."sidecarPath" is null+ )+ and "assets"."isVisible" = $2+-- AssetJobRepository.streamForDetectFacesJobselect"assets"."id"diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sqlindex a3dcb08c1e..432203c61d 100644--- a/server/src/queries/asset.repository.sql+++ b/server/src/queries/asset.repository.sql@@ -238,18 +238,13 @@ selectfrom"assets"where- (- "assets"."sidecarPath" = $1- or "assets"."sidecarPath" is null- )- and "assets"."isVisible" = $2- and "deletedAt" is null+ "deletedAt" is nullorder by"createdAt"limit- $3+ $1offset- $4+ $2-- AssetRepository.getTimeBucketswithdiff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.tsindex 78ddeae3e2..17756b2935 100644--- a/server/src/repositories/asset-job.repository.ts+++ b/server/src/repositories/asset-job.repository.ts@@ -323,6 +323,18 @@ export class AssetJobRepository {.stream();}+ @GenerateSql({ params: [], stream: true })+ streamForSidecar(force?: boolean) {+ return this.db+ .selectFrom('assets')+ .select(['assets.id'])+ .$if(!force, (qb) =>+ qb.where((eb) => eb.or([eb('assets.sidecarPath', '=', ''), eb('assets.sidecarPath', 'is', null)])),+ )+ .where('assets.isVisible', '=', true)+ .stream();+ }+@GenerateSql({ params: [], stream: true })streamForDetectFacesJob(force?: boolean) {return this.assetsWithPreviews()diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.tsindex ade6201cf1..7ba75db642 100644--- a/server/src/repositories/asset.repository.ts+++ b/server/src/repositories/asset.repository.ts@@ -48,7 +48,6 @@ export interface LivePhotoSearchOptions {export enum WithoutProperty {THUMBNAIL = 'thumbnail',ENCODED_VIDEO = 'encoded-video',- SIDECAR = 'sidecar',}export enum WithProperty {@@ -542,11 +541,6 @@ export class AssetRepository {.where((eb) => eb.or([eb('assets.encodedVideoPath', 'is', null), eb('assets.encodedVideoPath', '=', '')])),)- .$if(property === WithoutProperty.SIDECAR, (qb) =>- qb- .where((eb) => eb.or([eb('assets.sidecarPath', '=', ''), eb('assets.sidecarPath', 'is', null)]))- .where('assets.isVisible', '=', true),- ).$if(property === WithoutProperty.THUMBNAIL, (qb) =>qb.innerJoin('asset_job_status as job_status', 'assetId', 'assets.id')diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.tsindex dfa2140824..e0a283b02a 100644--- a/server/src/services/metadata.service.spec.ts+++ b/server/src/services/metadata.service.spec.ts@@ -5,7 +5,6 @@ import { constants } from 'node:fs/promises';import { defaults } from 'src/config';import { MapAsset } from 'src/dtos/asset-response.dto';import { AssetType, ExifOrientation, ImmichWorker, JobName, JobStatus, SourceType } from 'src/enum';-import { WithoutProperty } from 'src/repositories/asset.repository';import { ImmichTags } from 'src/repositories/metadata.repository';import { MetadataService } from 'src/services/metadata.service';import { assetStub } from 'test/fixtures/asset.stub';@@ -1346,12 +1345,11 @@ describe(MetadataService.name, () => {describe('handleQueueSidecar', () => {it('should queue assets with sidecar files', async () => {- mocks.asset.getAll.mockResolvedValue({ items: [assetStub.sidecar], hasNextPage: false });+ mocks.assetJob.streamForSidecar.mockReturnValue(makeStream([assetStub.image]));await sut.handleQueueSidecar({ force: true });+ expect(mocks.assetJob.streamForSidecar).toHaveBeenCalledWith(true);- expect(mocks.asset.getAll).toHaveBeenCalledWith({ take: 1000, skip: 0 });- expect(mocks.asset.getWithout).not.toHaveBeenCalled();expect(mocks.job.queueAll).toHaveBeenCalledWith([{name: JobName.SIDECAR_SYNC,@@ -1361,12 +1359,11 @@ describe(MetadataService.name, () => {});it('should queue assets without sidecar files', async () => {- mocks.asset.getWithout.mockResolvedValue({ items: [assetStub.image], hasNextPage: false });+ mocks.assetJob.streamForSidecar.mockReturnValue(makeStream([assetStub.image]));await sut.handleQueueSidecar({ force: false });- expect(mocks.asset.getWithout).toHaveBeenCalledWith({ take: 1000, skip: 0 }, WithoutProperty.SIDECAR);- expect(mocks.asset.getAll).not.toHaveBeenCalled();+ expect(mocks.assetJob.streamForSidecar).toHaveBeenCalledWith(false);expect(mocks.job.queueAll).toHaveBeenCalledWith([{name: JobName.SIDECAR_DISCOVERY,diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.tsindex 7c9a4c4b19..fd7382e163 100644--- a/server/src/services/metadata.service.ts+++ b/server/src/services/metadata.service.ts@@ -22,14 +22,12 @@ import {QueueName,SourceType,} from 'src/enum';-import { WithoutProperty } from 'src/repositories/asset.repository';import { ArgOf } from 'src/repositories/event.repository';import { ReverseGeocodeResult } from 'src/repositories/map.repository';import { ImmichTags } from 'src/repositories/metadata.repository';import { BaseService } from 'src/services/base.service';-import { JobOf } from 'src/types';+import { JobItem, JobOf } from 'src/types';import { isFaceImportEnabled } from 'src/utils/misc';-import { usePagination } from 'src/utils/pagination';import { upsertTags } from 'src/utils/tag';/** look for a date from these tags (in order) */@@ -289,23 +287,23 @@ export class MetadataService extends BaseService {}@OnJob({ name: JobName.QUEUE_SIDECAR, queue: QueueName.SIDECAR })- async handleQueueSidecar(job: JobOf<JobName.QUEUE_SIDECAR>): Promise<JobStatus> {- const { force } = job;- const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {- return force- ? this.assetRepository.getAll(pagination)- : this.assetRepository.getWithout(pagination, WithoutProperty.SIDECAR);- });+ async handleQueueSidecar({ force }: JobOf<JobName.QUEUE_SIDECAR>): Promise<JobStatus> {+ let jobs: JobItem[] = [];+ const queueAll = async () => {+ await this.jobRepository.queueAll(jobs);+ jobs = [];+ };- for await (const assets of assetPagination) {- await this.jobRepository.queueAll(- assets.map((asset) => ({- name: force ? JobName.SIDECAR_SYNC : JobName.SIDECAR_DISCOVERY,- data: { id: asset.id },- })),- );+ const assets = this.assetJobRepository.streamForSidecar(force);+ for await (const asset of assets) {+ jobs.push({ name: force ? JobName.SIDECAR_SYNC : JobName.SIDECAR_DISCOVERY, data: { id: asset.id } });+ if (jobs.length >= JOBS_ASSET_PAGINATION_SIZE) {+ await queueAll();+ }}+ await queueAll();+return JobStatus.SUCCESS;}