High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat(server): add OCR extraction workflow filter (#32464)
Valentin Agachi2e0eb35e93dc
diff --git a/i18n/en.json b/i18n/en.jsonindex 99193e5c84..17209e5d81 100644--- a/i18n/en.json+++ b/i18n/en.json@@ -2245,6 +2245,8 @@"trigger": "Trigger","trigger_asset_metadata_extraction": "Asset Metadata Extraction","trigger_asset_metadata_extraction_description": "Triggered when the EXIF metadata of an asset is extracted",+ "trigger_asset_ocr_extraction": "Asset OCR Extraction",+ "trigger_asset_ocr_extraction_description": "Triggered when the OCR text of an asset is extracted","trigger_asset_tagged": "Asset Tagged","trigger_asset_tagged_description": "Triggered when a tag is added to an asset","trigger_asset_uploaded": "Asset Upload",diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.jsonindex 6cf8848870..8d18513dc9 100644--- a/open-api/immich-openapi-specs.json+++ b/open-api/immich-openapi-specs.json@@ -34644,6 +34644,7 @@"enum": ["AssetCreate","AssetMetadataExtraction",+ "AssetOcrExtraction","AssetTagged"],"type": "string"diff --git a/packages/plugin-core/manifest.json b/packages/plugin-core/manifest.jsonindex ac8ba474d8..7866469e75 100644--- a/packages/plugin-core/manifest.json+++ b/packages/plugin-core/manifest.json@@ -75,6 +75,32 @@}],"uiHints": ["SmartAlbum"]+ },+ {+ "name": "receipts",+ "title": "Receipts",+ "description": "Automatically add receipt assets to an album",+ "trigger": "AssetOcrExtraction",+ "steps": [+ {+ "method": "immich-plugin-core#assetOcrFilter",+ "config": {+ "caseSensitive": false,+ "pattern": "receipt.+total",+ "matchType": "regex",+ "matchScope": "full",+ "minScore": 0+ }+ },+ {+ "method": "immich-plugin-core#assetAddToAlbums",+ "config": {+ "albumName": "Receipts",+ "albumIds": []+ }+ }+ ],+ "uiHints": ["SmartAlbum"]}],"methods": [@@ -428,6 +454,50 @@},"uiHints": ["Filter"]},+ {+ "name": "assetOcrFilter",+ "title": "Filter by OCR text",+ "description": "Filter assets by OCR text pattern using text matching or regular expressions",+ "types": ["AssetV1"],+ "schema": {+ "type": "object",+ "properties": {+ "pattern": {+ "type": "string",+ "title": "OCR text pattern",+ "description": "Text or regex pattern to match against OCR text"+ },+ "matchType": {+ "type": "string",+ "enum": ["contains", "startsWith", "exact", "regex"],+ "default": "contains",+ "title": "Match type",+ "description": "Type of pattern matching to perform"+ },+ "caseSensitive": {+ "type": "boolean",+ "default": false,+ "title": "Case sensitive",+ "description": "Whether matching should be case-sensitive"+ },+ "minScore": {+ "type": "number",+ "default": 0,+ "title": "Minimum score",+ "description": "Minimum OCR confidence score to match"+ },+ "matchScope": {+ "type": "string",+ "enum": ["full", "box"],+ "default": "full",+ "title": "Match scope",+ "description": "Scope of the OCR text to match against"+ }+ },+ "required": ["pattern"]+ },+ "uiHints": ["Filter"]+ },{"name": "assetArchive","title": "Archive asset",diff --git a/packages/plugin-core/src/index.ts b/packages/plugin-core/src/index.tsindex 915efbfb5e..95239e29ff 100644--- a/packages/plugin-core/src/index.ts+++ b/packages/plugin-core/src/index.ts@@ -194,6 +194,13 @@ const methods = wrapper<Manifest>({return { workflow: { continue: hasTimeZone === needsTimeZone } };},+ assetOcrFilter: ({ config, data }) => {+ const entries = (data.asset.ocr ?? []).filter((ocr) => ocr.textScore >= (config.minScore ?? 0));+ const texts = config.matchScope === 'full' ? [entries.map((ocr) => ocr.text).join(' ')] : entries.map((ocr) => ocr.text);+ const matched = texts.some((t) => matchValueResult(t, config).workflow?.continue);+ return { workflow: { continue: matched } };+ },+assetTagFilter: ({ config, data }) => {const assetTags = data.asset.tags.map((tag) => tag.id);@@ -254,6 +261,7 @@ const {assetDateFilter,assetLock,assetMissingTimeZoneFilter,+ assetOcrFilter,assetTagFilter,assetTypeFilter,assetVisibility,@@ -274,6 +282,7 @@ export {assetDateFilter,assetLock,assetMissingTimeZoneFilter,+ assetOcrFilter,assetTagFilter,assetTypeFilter,assetVisibility,diff --git a/packages/plugin-sdk/src/types.ts b/packages/plugin-sdk/src/types.tsindex 94c968978d..23092de966 100644--- a/packages/plugin-sdk/src/types.ts+++ b/packages/plugin-sdk/src/types.ts@@ -23,6 +23,7 @@ export type WorkflowEventData<T extends WorkflowType> = WorkflowEventMap[T];export enum WorkflowTrigger {AssetCreate = 'AssetCreate',AssetMetadataExtraction = 'AssetMetadataExtraction',+ AssetOcrExtraction = 'AssetOcrExtraction',AssetTagged = 'AssetTagged',// PersonRecognized = 'PersonRecognized',}@@ -127,6 +128,11 @@ export type AssetV1 = {tags: string[] | null;updatedAt: string | null;} | null;+ ocr?: {+ text: string;+ textScore: number;+ boxScore: number;+ }[]};};diff --git a/packages/sdk/src/fetch-client.ts b/packages/sdk/src/fetch-client.tsindex 6e870cad8c..ee0f0de0a9 100644--- a/packages/sdk/src/fetch-client.ts+++ b/packages/sdk/src/fetch-client.ts@@ -8603,6 +8603,7 @@ export enum WorkflowType {export enum WorkflowTrigger {AssetCreate = "AssetCreate",AssetMetadataExtraction = "AssetMetadataExtraction",+ AssetOcrExtraction = "AssetOcrExtraction",AssetTagged = "AssetTagged"}export enum QueueJobStatus {diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sqlindex 3c3a97eb48..1e0cf6c191 100644--- a/server/src/queries/asset.job.repository.sql+++ b/server/src/queries/asset.job.repository.sql@@ -615,6 +615,7 @@ where-- AssetJobRepository.getForOcrselect+ "asset"."ownerId","asset"."visibility",(selectdiff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.tsindex 9a71c98a08..e1131b7cef 100644--- a/server/src/repositories/asset-job.repository.ts+++ b/server/src/repositories/asset-job.repository.ts@@ -255,7 +255,7 @@ export class AssetJobRepository {getForOcr(id: string) {return this.db.selectFrom('asset')- .select((eb) => ['asset.visibility', withFilePath(eb, AssetFileType.Preview).as('previewFile')])+ .select((eb) => ['asset.ownerId', 'asset.visibility', withFilePath(eb, AssetFileType.Preview).as('previewFile')]).where('asset.id', '=', id).executeTakeFirst();}diff --git a/server/src/repositories/event.repository.ts b/server/src/repositories/event.repository.tsindex a1c6a42617..60a227a183 100644--- a/server/src/repositories/event.repository.ts+++ b/server/src/repositories/event.repository.ts@@ -53,6 +53,7 @@ type EventMap = {AssetTrash: [{ assetId: string; userId: string }];AssetDelete: [{ assetId: string; userId: string }];AssetMetadataExtracted: [{ assetId: string; userId: string; source?: JobSource }];+ AssetOcrExtracted: [{ assetId: string; userId: string }];// asset bulk eventsAssetTrashAll: [{ assetIds: string[]; userId: string }];diff --git a/server/src/repositories/workflow.repository.ts b/server/src/repositories/workflow.repository.tsindex b0c0c932cc..7a7147d382 100644--- a/server/src/repositories/workflow.repository.ts+++ b/server/src/repositories/workflow.repository.ts@@ -1,3 +1,4 @@+import { WorkflowTrigger } from '@immich/plugin-sdk';import { Injectable } from '@nestjs/common';import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';import { InjectKysely } from 'nestjs-kysely';@@ -177,7 +178,7 @@ export class WorkflowRepository {await this.db.deleteFrom('workflow').where('id', '=', id).execute();}- getForAssetV1(assetId: string) {+ getForAssetV1(assetId: string, trigger: WorkflowTrigger) {return this.db.selectFrom('asset').leftJoin('asset_exif', 'asset_exif.assetId', 'asset.id')@@ -221,6 +222,17 @@ export class WorkflowRepository {]).whereRef('asset_exif.assetId', '=', 'asset.id'),).as('exifInfo'),+ ...(trigger === WorkflowTrigger.AssetOcrExtraction+ ? [+ jsonArrayFrom(+ eb+ .selectFrom('asset_ocr')+ .select(['asset_ocr.text', 'asset_ocr.textScore', 'asset_ocr.boxScore'])+ .whereRef('asset_ocr.assetId', '=', 'asset.id')+ .where('isVisible', '=', true),+ ).as('ocr'),+ ]+ : []),]).where('id', '=', assetId).executeTakeFirstOrThrow();diff --git a/server/src/services/ocr.service.spec.ts b/server/src/services/ocr.service.spec.tsindex 9e6ff9a051..4ef2f61e14 100644--- a/server/src/services/ocr.service.spec.ts+++ b/server/src/services/ocr.service.spec.ts@@ -13,6 +13,7 @@ describe(OcrService.name, () => {mocks.config.getWorker.mockReturnValue(ImmichWorker.Microservices);mocks.assetJob.getForOcr.mockResolvedValue({+ ownerId: 'user-id',visibility: AssetVisibility.Timeline,previewFile: '/uploads/user-id/thumbs/path.jpg',});@@ -69,16 +70,22 @@ describe(OcrService.name, () => {expect(mocks.asset.getByIds).not.toHaveBeenCalled();expect(mocks.machineLearning.encodeImage).not.toHaveBeenCalled();+ expect(mocks.event.emit).not.toHaveBeenCalled();});it('should skip assets without a resize path', async () => {const asset = AssetFactory.create();- mocks.assetJob.getForOcr.mockResolvedValue({ visibility: AssetVisibility.Timeline, previewFile: null });+ mocks.assetJob.getForOcr.mockResolvedValue({+ ownerId: 'userId',+ visibility: AssetVisibility.Timeline,+ previewFile: null,+ });expect(await sut.handleOcr({ id: asset.id })).toEqual(JobStatus.Failed);expect(mocks.ocr.upsert).not.toHaveBeenCalled();expect(mocks.machineLearning.ocr).not.toHaveBeenCalled();+ expect(mocks.event.emit).not.toHaveBeenCalled();});it('should save the returned objects', async () => {@@ -170,6 +177,7 @@ describe(OcrService.name, () => {it('should skip invisible assets', async () => {const asset = AssetFactory.from().file({ type: AssetFileType.Preview }).build();mocks.assetJob.getForOcr.mockResolvedValue({+ ownerId: 'userId',visibility: AssetVisibility.Hidden,previewFile: asset.files[0].path,});@@ -178,6 +186,7 @@ describe(OcrService.name, () => {expect(mocks.machineLearning.ocr).not.toHaveBeenCalled();expect(mocks.ocr.upsert).not.toHaveBeenCalled();+ expect(mocks.event.emit).not.toHaveBeenCalled();});it('should fail if asset could not be found', async () => {@@ -187,6 +196,19 @@ describe(OcrService.name, () => {expect(mocks.machineLearning.ocr).not.toHaveBeenCalled();expect(mocks.ocr.upsert).not.toHaveBeenCalled();+ expect(mocks.event.emit).not.toHaveBeenCalled();+ });++ it('should emit an event', async () => {+ const asset = AssetFactory.create();+ mockOcrResult('Hello World');++ await sut.handleOcr({ id: asset.id });++ expect(mocks.event.emit).toHaveBeenCalledWith('AssetOcrExtracted', {+ userId: 'user-id',+ assetId: asset.id,+ });});describe('search tokenization', () => {diff --git a/server/src/services/ocr.service.ts b/server/src/services/ocr.service.tsindex c91524e564..312076228e 100644--- a/server/src/services/ocr.service.ts+++ b/server/src/services/ocr.service.ts@@ -50,6 +50,11 @@ export class OcrService extends BaseService {await this.assetRepository.upsertJobStatus({ assetId: id, ocrAt: new Date() });+ await this.eventRepository.emit('AssetOcrExtracted', {+ userId: asset.ownerId,+ assetId: id,+ });+this.logger.debug(`Processed ${ocrResults.text.length} OCR result(s) for ${id}`);return JobStatus.Success;}diff --git a/server/src/services/workflow-execution.service.ts b/server/src/services/workflow-execution.service.tsindex b8149aa9b2..2185bfa8df 100644--- a/server/src/services/workflow-execution.service.ts+++ b/server/src/services/workflow-execution.service.ts@@ -316,6 +316,11 @@ export class WorkflowExecutionService extends BaseService {return this.onAssetTrigger({ userId, assetId, trigger: WorkflowTrigger.AssetMetadataExtraction });}+ @OnEvent({ name: 'AssetOcrExtracted' })+ onAssetOcrExtracted({ userId, assetId }: ArgOf<'AssetOcrExtracted'>) {+ return this.onAssetTrigger({ userId, assetId, trigger: WorkflowTrigger.AssetOcrExtraction });+ }+@OnEvent({ name: 'AssetTag' })onAssetTagged({ assetId, userId }: ArgOf<'AssetTag'>) {return this.onAssetTrigger({ userId, assetId, trigger: WorkflowTrigger.AssetTagged });@@ -332,7 +337,7 @@ export class WorkflowExecutionService extends BaseService {}@OnJob({ name: JobName.WorkflowAssetTrigger, queue: QueueName.Workflow })- handleAssetTrigger({ workflowId, assetId }: JobOf<JobName.WorkflowAssetTrigger>) {+ handleAssetTrigger({ workflowId, assetId, trigger }: JobOf<JobName.WorkflowAssetTrigger>) {return this.execute(workflowId, (type) => {const assetService = BaseService.create(AssetService, this);@@ -340,7 +345,7 @@ export class WorkflowExecutionService extends BaseService {case WorkflowType.AssetV1: {return {read: async () => {- const asset = await this.workflowRepository.getForAssetV1(assetId);+ const asset = await this.workflowRepository.getForAssetV1(assetId, trigger);return {data: { asset } as any,authUserId: asset.ownerId,diff --git a/server/src/types.ts b/server/src/types.tsindex 2dcf2ecc4a..d6310dc404 100644--- a/server/src/types.ts+++ b/server/src/types.ts@@ -1,3 +1,4 @@+import { WorkflowTrigger } from '@immich/plugin-sdk';import { ShallowDehydrateObject } from 'kysely';import { Mocked } from 'vitest';import { VECTOR_EXTENSIONS } from 'src/constants.js';@@ -439,7 +440,7 @@ export type JobItem =| { name: JobName.Ocr; data: IEntityJob }// Workflow- | { name: JobName.WorkflowAssetTrigger; data: { workflowId: string; assetId: string } }+ | { name: JobName.WorkflowAssetTrigger; data: { workflowId: string; assetId: string; trigger: WorkflowTrigger } }// Integrity| { name: JobName.IntegrityUntrackedFilesQueueAll; data?: IIntegrityJob }diff --git a/server/src/utils/workflow.ts b/server/src/utils/workflow.tsindex c763eec2d4..5de7100daf 100644--- a/server/src/utils/workflow.ts+++ b/server/src/utils/workflow.ts@@ -6,6 +6,7 @@ export const triggerMap: Record<WorkflowTrigger, WorkflowType[]> = {[WorkflowTrigger.AssetCreate]: [WorkflowType.AssetV1],// [WorkflowTrigger.PersonRecognized]: [WorkflowType.AssetPersonV1],[WorkflowTrigger.AssetMetadataExtraction]: [WorkflowType.AssetV1],+ [WorkflowTrigger.AssetOcrExtraction]: [WorkflowType.AssetV1],[WorkflowTrigger.AssetTagged]: [WorkflowType.AssetV1],};diff --git a/server/test/medium.factory.ts b/server/test/medium.factory.tsindex 881a0f6fbe..9dcde56e82 100644--- a/server/test/medium.factory.ts+++ b/server/test/medium.factory.ts@@ -78,6 +78,7 @@ import { AssetExifTable } from 'src/schema/tables/asset-exif.table.js';import { AssetFileTable } from 'src/schema/tables/asset-file.table.js';import { AssetJobStatusTable } from 'src/schema/tables/asset-job-status.table.js';import { AssetMetadataTable } from 'src/schema/tables/asset-metadata.table.js';+import { AssetOcrTable } from 'src/schema/tables/asset-ocr.table.js';import { AssetTable } from 'src/schema/tables/asset.table.js';import { FaceSearchTable } from 'src/schema/tables/face-search.table.js';import { MemoryTable } from 'src/schema/tables/memory.table.js';@@ -230,6 +231,28 @@ export class MediumTestContext<S extends ClassConstructor<typeof BaseService> =return { assetFace, result };}+ async newAssetOcr(+ dto: (Partial<Insertable<AssetOcrTable>> & { assetId: string; text: string; textScore: number })[],+ ) {+ const result = await this.get(OcrRepository).upsert(+ dto[0].assetId,+ dto.map((item) => ({+ ...item,+ x1: 0,+ y1: 0,+ x2: 1,+ y2: 1,+ x3: 2,+ y3: 2,+ x4: 3,+ y4: 3,+ boxScore: item.textScore,+ })),+ '',+ );+ return { ocr: dto, result };+ }+async newMemory(dto: Partial<Insertable<MemoryTable>> = {}) {const memory = mediumFactory.memoryInsert(dto);const result = await this.get(MemoryRepository).create(memory, new Set<string>());diff --git a/server/test/medium/specs/services/ocr.service.spec.ts b/server/test/medium/specs/services/ocr.service.spec.tsindex 16ae3b9d5d..837ae63fef 100644--- a/server/test/medium/specs/services/ocr.service.spec.ts+++ b/server/test/medium/specs/services/ocr.service.spec.ts@@ -3,6 +3,7 @@ import { AssetFileType, JobStatus } from 'src/enum.js';import { AssetJobRepository } from 'src/repositories/asset-job.repository.js';import { AssetRepository } from 'src/repositories/asset.repository.js';import { ConfigRepository } from 'src/repositories/config.repository.js';+import { EventRepository } from 'src/repositories/event.repository.js';import { JobRepository } from 'src/repositories/job.repository.js';import { LoggingRepository } from 'src/repositories/logging.repository.js';import { MachineLearningRepository } from 'src/repositories/machine-learning.repository.js';@@ -20,7 +21,7 @@ const setup = (db?: Kysely<DB>) => {return newMediumService(OcrService, {database: db || defaultDatabase,real: [AssetRepository, AssetJobRepository, ConfigRepository, OcrRepository, SystemMetadataRepository],- mock: [JobRepository, LoggingRepository, MachineLearningRepository],+ mock: [EventRepository, JobRepository, LoggingRepository, MachineLearningRepository],});};@@ -39,6 +40,7 @@ describe(OcrService.name, () => {const { user } = await ctx.newUser();const { asset } = await ctx.newAsset({ ownerId: user.id });await ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Preview, path: 'preview.jpg' });+ ctx.getMock(EventRepository).emit.mockResolvedValue();const machineLearningMock = ctx.getMock(MachineLearningRepository);machineLearningMock.ocr.mockResolvedValue({@@ -91,6 +93,7 @@ describe(OcrService.name, () => {const { user } = await ctx.newUser();const { asset } = await ctx.newAsset({ ownerId: user.id });await ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Preview, path: 'preview.jpg' });+ ctx.getMock(EventRepository).emit.mockResolvedValue();const machineLearningMock = ctx.getMock(MachineLearningRepository);machineLearningMock.ocr.mockResolvedValue({@@ -215,6 +218,7 @@ describe(OcrService.name, () => {const { user } = await ctx.newUser();const { asset } = await ctx.newAsset({ ownerId: user.id });await ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Preview, path: 'preview.jpg' });+ ctx.getMock(EventRepository).emit.mockResolvedValue();const machineLearningMock = ctx.getMock(MachineLearningRepository);machineLearningMock.ocr.mockResolvedValue({ box: [], boxScore: [], text: [], textScore: [] });@@ -240,6 +244,7 @@ describe(OcrService.name, () => {const { user } = await ctx.newUser();const { asset } = await ctx.newAsset({ ownerId: user.id });await ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Preview, path: 'preview.jpg' });+ ctx.getMock(EventRepository).emit.mockResolvedValue();const machineLearningMock = ctx.getMock(MachineLearningRepository);machineLearningMock.ocr.mockResolvedValue({diff --git a/server/test/medium/specs/workflow/workflow-core-plugin.spec.ts b/server/test/medium/specs/workflow/workflow-core-plugin.spec.tsindex e4501f3172..6e53c25139 100644--- a/server/test/medium/specs/workflow/workflow-core-plugin.spec.ts+++ b/server/test/medium/specs/workflow/workflow-core-plugin.spec.ts@@ -146,7 +146,13 @@ describe('core plugin', () => {steps: [{ method: 'immich-plugin-core#assetArchive' }],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({visibility: AssetVisibility.Archive,@@ -163,7 +169,13 @@ describe('core plugin', () => {steps: [{ method: 'immich-plugin-core#assetArchive', config: { inverse: true } }],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({visibility: AssetVisibility.Timeline,@@ -182,7 +194,13 @@ describe('core plugin', () => {steps: [{ method: 'immich-plugin-core#assetLock' }],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({visibility: AssetVisibility.Locked,@@ -199,7 +217,13 @@ describe('core plugin', () => {steps: [{ method: 'immich-plugin-core#assetLock', config: { inverse: true } }],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({visibility: AssetVisibility.Timeline,@@ -218,7 +242,13 @@ describe('core plugin', () => {steps: [{ method: 'immich-plugin-core#assetFavorite' }],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: true });});@@ -233,7 +263,13 @@ describe('core plugin', () => {steps: [{ method: 'immich-plugin-core#assetFavorite', config: { inverse: true } }],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: false });});@@ -250,7 +286,13 @@ describe('core plugin', () => {steps: [{ method: 'immich-plugin-core#assetAddToAlbums', config: { albumIds: [], albumName: 'Screenshots' } }],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();const albums = await ctx.get(AlbumRepository).getAll(user.id);expect(albums).toHaveLength(1);@@ -281,7 +323,13 @@ describe('core plugin', () => {expect(albums).toHaveLength(1);expect(albums[0].albumName).toEqual(album.albumName);- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AlbumRepository).getAssetIds(album.id, [asset.id])).resolves.toContain(asset.id);});@@ -297,7 +345,13 @@ describe('core plugin', () => {steps: [{ method: 'immich-plugin-core#assetAddToAlbums', config: { albumIds: [album.id] } }],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AlbumRepository).getAssetIds(album.id, [asset.id])).resolves.toContain(asset.id);});@@ -316,7 +370,13 @@ describe('core plugin', () => {steps: [{ method: 'immich-plugin-core#assetAddToAlbums', config: { albumIds: [album1.id, album2.id] } }],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AlbumRepository).getAssetIds(album1.id, [asset.id])).resolves.toContain(asset.id);await expect(ctx.get(AlbumRepository).getAssetIds(album2.id, [asset.id])).resolves.toContain(asset.id);@@ -334,7 +394,13 @@ describe('core plugin', () => {steps: [{ method: 'immich-plugin-core#assetAddToAlbums', config: { albumIds: [album.id] } }],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeTruthy();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeTruthy();await expect(ctx.get(AlbumRepository).getAssetIds(album.id, [asset.id])).resolves.not.toContain(asset.id);});@@ -360,7 +426,13 @@ describe('core plugin', () => {],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: true });});@@ -383,7 +455,13 @@ describe('core plugin', () => {],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetMetadataExtraction,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: false });});@@ -406,7 +484,13 @@ describe('core plugin', () => {],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetMetadataExtraction,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: true });});});@@ -434,10 +518,18 @@ describe('core plugin', () => {});await expect(- ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset1.id }),+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset1.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),).resolves.toBeUndefined();await expect(- ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset2.id }),+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset2.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true });await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: true });@@ -465,10 +557,18 @@ describe('core plugin', () => {});await expect(- ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset1.id }),+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset1.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),).resolves.toBeUndefined();await expect(- ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset2.id }),+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset2.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true });await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: false });@@ -496,10 +596,18 @@ describe('core plugin', () => {});await expect(- ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset1.id }),+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset1.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),).resolves.toBeUndefined();await expect(- ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset2.id }),+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset2.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true });await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: false });@@ -525,7 +633,13 @@ describe('core plugin', () => {],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: true });});});@@ -663,13 +777,25 @@ describe('core plugin', () => {],});- await ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset1.id });+ await ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset1.id,+ trigger: WorkflowTrigger.AssetCreate,+ });await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true });- await ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset2.id });+ await ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset2.id,+ trigger: WorkflowTrigger.AssetCreate,+ });await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: true });- await ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset3.id });+ await ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset3.id,+ trigger: WorkflowTrigger.AssetCreate,+ });await expect(ctx.get(AssetRepository).getById(asset3.id)).resolves.toMatchObject({ isFavorite: false });});@@ -700,17 +826,157 @@ describe('core plugin', () => {],});- await ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset1.id });+ await ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset1.id,+ trigger: WorkflowTrigger.AssetCreate,+ });await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true });- await ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset2.id });+ await ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset2.id,+ trigger: WorkflowTrigger.AssetCreate,+ });await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: true });- await ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset3.id });+ await ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset3.id,+ trigger: WorkflowTrigger.AssetCreate,+ });await expect(ctx.get(AssetRepository).getById(asset3.id)).resolves.toMatchObject({ isFavorite: false });});});+ describe('assetOcrFilter', () => {+ it('should match assets case-insensitively', async () => {+ const { user } = await ctx.newUser();+ const [{ asset: asset1 }, { asset: asset2 }] = await Promise.all([+ ctx.newAsset({ ownerId: user.id }),+ ctx.newAsset({ ownerId: user.id }),+ ]);+ await ctx.newAssetOcr([{ assetId: asset1.id, text: 'test', textScore: 0.9 }]);+ await ctx.newAssetOcr([{ assetId: asset2.id, text: 'bogus', textScore: 0.9 }]);++ const workflow = await createWorkflow({+ ownerId: user.id,+ trigger: WorkflowTrigger.AssetOcrExtraction,+ steps: [+ {+ method: 'immich-plugin-core#assetOcrFilter',+ config: { matchType: 'contains', matchScope: 'box', pattern: 'Test', minScore: 0 },+ },+ {+ method: 'immich-plugin-core#assetFavorite',+ },+ ],+ });++ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset1.id,+ trigger: WorkflowTrigger.AssetOcrExtraction,+ }),+ ).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset2.id,+ trigger: WorkflowTrigger.AssetOcrExtraction,+ }),+ ).resolves.toBeUndefined();+ await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true });+ await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: false });+ });++ it('should match assets by regex in full text', async () => {+ const { user } = await ctx.newUser();+ const [{ asset: asset1 }, { asset: asset2 }] = await Promise.all([+ ctx.newAsset({ ownerId: user.id }),+ ctx.newAsset({ ownerId: user.id }),+ ]);+ await ctx.newAssetOcr([+ { assetId: asset1.id, text: 'Test', textScore: 0.9 },+ { assetId: asset1.id, text: 'Match', textScore: 0.5 },+ ]);+ await ctx.newAssetOcr([{ assetId: asset2.id, text: 'Test', textScore: 0.9 }]);++ const workflow = await createWorkflow({+ ownerId: user.id,+ trigger: WorkflowTrigger.AssetCreate,+ steps: [+ {+ method: 'immich-plugin-core#assetOcrFilter',+ config: { matchType: 'regex', matchScope: 'full', pattern: 'Test.+Match', minScore: 0 },+ },+ {+ method: 'immich-plugin-core#assetFavorite',+ },+ ],+ });++ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset1.id,+ trigger: WorkflowTrigger.AssetOcrExtraction,+ }),+ ).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset2.id,+ trigger: WorkflowTrigger.AssetOcrExtraction,+ }),+ ).resolves.toBeUndefined();+ await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true });+ await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: false });+ });++ it('should filter assets with low score', async () => {+ const { user } = await ctx.newUser();+ const [{ asset: asset1 }, { asset: asset2 }] = await Promise.all([+ ctx.newAsset({ ownerId: user.id }),+ ctx.newAsset({ ownerId: user.id }),+ ]);+ await ctx.newAssetOcr([{ assetId: asset1.id, text: 'test', textScore: 0.9 }]);+ await ctx.newAssetOcr([{ assetId: asset2.id, text: 'test', textScore: 0.5 }]);++ const workflow = await createWorkflow({+ ownerId: user.id,+ trigger: WorkflowTrigger.AssetOcrExtraction,+ steps: [+ {+ method: 'immich-plugin-core#assetOcrFilter',+ config: { matchType: 'contains', matchScope: 'box', pattern: 'test', minScore: 0.8 },+ },+ {+ method: 'immich-plugin-core#assetFavorite',+ },+ ],+ });++ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset1.id,+ trigger: WorkflowTrigger.AssetOcrExtraction,+ }),+ ).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset2.id,+ trigger: WorkflowTrigger.AssetOcrExtraction,+ }),+ ).resolves.toBeUndefined();+ await expect(ctx.get(AssetRepository).getById(asset1.id)).resolves.toMatchObject({ isFavorite: true });+ await expect(ctx.get(AssetRepository).getById(asset2.id)).resolves.toMatchObject({ isFavorite: false });+ });+ });+describe('assetTagFilter', () => {it('should favorite asset if it matches any tag', async () => {const { user } = await ctx.newUser();@@ -732,7 +998,13 @@ describe('core plugin', () => {],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: true });});@@ -755,7 +1027,13 @@ describe('core plugin', () => {],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: false });});@@ -779,7 +1057,13 @@ describe('core plugin', () => {],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: true });});@@ -802,7 +1086,13 @@ describe('core plugin', () => {],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: false });});@@ -826,7 +1116,13 @@ describe('core plugin', () => {],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: true });});});@@ -850,7 +1146,13 @@ describe('core plugin', () => {],});- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();expect(fetchMock).toHaveBeenCalled();});@@ -887,7 +1189,13 @@ describe('core plugin', () => {.execute();}- await expect(ctx.sut.handleAssetTrigger({ workflowId: workflow.id, assetId: asset.id })).resolves.toBeUndefined();+ await expect(+ ctx.sut.handleAssetTrigger({+ workflowId: workflow.id,+ assetId: asset.id,+ trigger: WorkflowTrigger.AssetCreate,+ }),+ ).resolves.toBeUndefined();await expect(ctx.get(AssetRepository).getById(asset.id)).resolves.toMatchObject({ isFavorite: false });});});diff --git a/web/src/lib/utils/workflow.ts b/web/src/lib/utils/workflow.tsindex 9aff59153c..66234d7726 100644--- a/web/src/lib/utils/workflow.ts+++ b/web/src/lib/utils/workflow.ts@@ -13,6 +13,9 @@ export const getTriggerName = ($t: MessageFormatter, type: WorkflowTrigger) => {case WorkflowTrigger.AssetMetadataExtraction: {return $t('trigger_asset_metadata_extraction');}+ case WorkflowTrigger.AssetOcrExtraction: {+ return $t('trigger_asset_ocr_extraction');+ }case WorkflowTrigger.AssetTagged: {return $t('trigger_asset_tagged');}@@ -33,6 +36,9 @@ export const getTriggerDescription = ($t: MessageFormatter, type: WorkflowTriggecase WorkflowTrigger.AssetMetadataExtraction: {return $t('trigger_asset_metadata_extraction_description');}+ case WorkflowTrigger.AssetOcrExtraction: {+ return $t('trigger_asset_ocr_extraction_description');+ }case WorkflowTrigger.AssetTagged: {return $t('trigger_asset_tagged_description');}