High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
fix: library watcher should respect ignore patterns (#31418)
Jonathan Jogenfors76c08fa230b8
diff --git a/server/src/repositories/storage.repository.spec.ts b/server/src/repositories/storage.repository.spec.tsindex aca530835a..3d1e503ed4 100644--- a/server/src/repositories/storage.repository.spec.ts+++ b/server/src/repositories/storage.repository.spec.ts@@ -1,9 +1,30 @@import mockfs from 'mock-fs';+import { vitest } from 'vitest';import { CrawlOptionsDto } from 'src/dtos/library.dto.js';import { LoggingRepository } from 'src/repositories/logging.repository.js';import { StorageRepository } from 'src/repositories/storage.repository.js';import { automock } from 'test/utils.js';+const mocks = vitest.hoisted(() => {+ const watcher = {+ close: vitest.fn(),+ on: vitest.fn(),+ };+ watcher.on.mockReturnValue(watcher);++ return { watch: vitest.fn(() => watcher), watcher };+});++vitest.mock('chokidar', () => ({ watch: mocks.watch }));++const getHandler = (event: string) => {+ const handler = mocks.watcher.on.mock.calls.find(([name]) => name === event)?.[1];+ if (!handler) {+ throw new Error(`Missing ${event} handler`);+ }+ return handler;+};+interface Test {test: string;options: CrawlOptionsDto;@@ -185,6 +206,9 @@ describe(StorageRepository.name, () => {beforeEach(() => {// eslint-disable-next-line no-sparse-arrayssut = new StorageRepository(automock(LoggingRepository, { args: [, { getEnv: () => ({}) }], strict: false }));+ mocks.watch.mockReset().mockReturnValue(mocks.watcher);+ mocks.watcher.on.mockReset().mockReturnValue(mocks.watcher);+ mocks.watcher.close.mockReset().mockResolvedValue(undefined);});afterEach(() => {@@ -206,4 +230,55 @@ describe(StorageRepository.name, () => {});}});++ describe('watch', () => {+ it('should register event handlers and close the watcher', async () => {+ const onReady = vitest.fn();+ const onAdd = vitest.fn();+ const onChange = vitest.fn();+ const onUnlink = vitest.fn();+ const onError = vitest.fn();+ const close = sut.watch(['/photos'], {}, { onAdd, onChange, onError, onReady, onUnlink });++ expect(mocks.watch).toHaveBeenCalledWith(['/photos'], expect.objectContaining({ ignored: expect.any(Function) }));++ const error = new Error('watch error');+ getHandler('ready')();+ getHandler('add')('/photos/add.jpg');+ getHandler('change')('/photos/change.jpg');+ getHandler('unlink')('/photos/unlink.jpg');+ getHandler('error')(error);++ expect(onReady).toHaveBeenCalledWith();+ expect(onAdd).toHaveBeenCalledWith('/photos/add.jpg');+ expect(onChange).toHaveBeenCalledWith('/photos/change.jpg');+ expect(onUnlink).toHaveBeenCalledWith('/photos/unlink.jpg');+ expect(onError).toHaveBeenCalledWith(error);++ await close();+ expect(mocks.watcher.close).toHaveBeenCalledWith();+ });++ it('should convert ignored glob arrays to a case-insensitive matcher', () => {+ sut.watch(['/photos'], { ignored: ['**/excluded/**'] }, {});+ const [, options] = mocks.watch.mock.lastCall as unknown as [string[], { ignored?: unknown }];+ const ignored = options.ignored as (path: string) => boolean;++ expect(typeof ignored).toBe('function');+ expect(ignored('/photos/EXCLUDED/photo.jpg')).toBe(true);+ expect(ignored('/photos/included/photo.jpg')).toBe(false);+ });++ it('should tolerate missing event callbacks', () => {+ sut.watch(['/photos'], {}, {});++ expect(() => {+ getHandler('ready')();+ getHandler('add')('/photos/add.jpg');+ getHandler('change')('/photos/change.jpg');+ getHandler('unlink')('/photos/unlink.jpg');+ getHandler('error')(new Error('watch error'));+ }).not.toThrow();+ });+ });});diff --git a/server/src/repositories/storage.repository.ts b/server/src/repositories/storage.repository.tsindex 8464f19ec1..eafc3346bd 100644--- a/server/src/repositories/storage.repository.ts+++ b/server/src/repositories/storage.repository.ts@@ -16,6 +16,7 @@ import fs from 'node:fs/promises';import path from 'node:path';import { PassThrough, Readable, Writable } from 'node:stream';import { createGunzip, createGzip } from 'node:zlib';+import picomatch from 'picomatch';import { CrawlOptionsDto, WalkOptionsDto } from 'src/dtos/library.dto.js';import { LoggingRepository } from 'src/repositories/logging.repository.js';import { mimeTypes } from 'src/utils/mime-types.js';@@ -28,6 +29,8 @@ export interface WatchEvents {onError(error: Error): void;}+export type WatchOptions = Omit<ChokidarOptions, 'ignored'> & { ignored?: string[] };+export interface ImmichReadStream {stream: Readable;type?: string;@@ -273,8 +276,15 @@ export class StorageRepository {}}- watch(paths: string[], options: ChokidarOptions, events: Partial<WatchEvents>) {- const watcher = chokidarWatch(paths, options);+ watch(paths: string[], options: WatchOptions, events: Partial<WatchEvents>) {+ const matchesIgnoredPath = picomatch(options.ignored ?? [], {+ dot: true, // Match the behavior of fast-glob's micromatch by using these settings+ nocase: true,+ posix: true,+ strictSlashes: false,+ });++ const watcher = chokidarWatch(paths, { ...options, ignored: (path) => matchesIgnoredPath(path) });watcher.on('ready', () => events.onReady?.());watcher.on('add', (path) => events.onAdd?.(path));diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.tsindex 83b1a123f2..53b3e7035c 100644--- a/server/src/services/library.service.spec.ts+++ b/server/src/services/library.service.spec.ts@@ -942,6 +942,24 @@ describe(LibraryService.name, () => {expect(mocks.storage.watch).toHaveBeenCalledWith(library.importPaths, expect.anything(), expect.anything());});+ it('should exclude paths from the watcher', async () => {+ const library = factory.library({+ importPaths: ['/foo', '/bar'],+ exclusionPatterns: ['**/excluded/**'],+ });++ mocks.library.get.mockResolvedValue(library);+ mocks.library.getAll.mockResolvedValue([library]);++ await sut.watchAll();++ expect(mocks.storage.watch).toHaveBeenCalledWith(+ library.importPaths,+ expect.objectContaining({ ignored: library.exclusionPatterns }),+ expect.anything(),+ );+ });+it('should watch and unwatch library', async () => {const library = factory.library({ importPaths: ['/foo', '/bar'] });diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.tsindex d86589a181..1a8fb35569 100644--- a/server/src/services/library.service.ts+++ b/server/src/services/library.service.ts@@ -135,6 +135,7 @@ export class LibraryService extends BaseService {{usePolling: false,ignoreInitial: true,+ ignored: library.exclusionPatterns,awaitWriteFinish: {stabilityThreshold: 5000,pollInterval: 1000,diff --git a/server/test/medium/specs/repositories/storage.repository.spec.ts b/server/test/medium/specs/repositories/storage.repository.spec.tsnew file mode 100644index 0000000000..bf34c5c256--- /dev/null+++ b/server/test/medium/specs/repositories/storage.repository.spec.ts@@ -0,0 +1,88 @@+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { LoggingRepository } from 'src/repositories/logging.repository.js';+import { StorageRepository, type WatchOptions } from 'src/repositories/storage.repository.js';+import { automock } from 'test/utils.js';++describe(StorageRepository.name, () => {+ const annoyingExclusionPatterns = ['@', '#', '$', '%', '^', '&', '='];+ let sut: StorageRepository;+ let tempDir: string;+ let includedFile: string;+ let excludedFile: string;++ beforeEach(async () => {+ // eslint-disable-next-line no-sparse-arrays+ sut = new StorageRepository(automock(LoggingRepository, { args: [, { getEnv: () => ({}) }], strict: false }));+ tempDir = await mkdtemp(join(tmpdir(), 'immich-storage-watch-'));+ includedFile = join(tempDir, 'included/photo.jpg');+ excludedFile = join(tempDir, 'excluded/photo.jpg');++ await Promise.all([+ mkdir(join(tempDir, 'included'), { recursive: true }),+ mkdir(join(tempDir, 'excluded'), { recursive: true }),+ ]);+ await Promise.all([writeFile(includedFile, 'included'), writeFile(excludedFile, 'excluded')]);+ });++ afterEach(async () => {+ await rm(tempDir, { recursive: true, force: true });+ });++ const startWatching = (options: WatchOptions) => {+ const paths: string[] = [];+ let resolveReady!: () => void;++ const ready = new Promise<void>((resolve) => (resolveReady = resolve));+ const close = sut.watch([tempDir], options, {+ onAdd: (path) => {+ paths.push(path);+ },+ onReady: resolveReady,+ });++ return { close, paths, ready };+ };++ it('should emit add events for initial files', async () => {+ const watcher = startWatching({ ignoreInitial: false });++ try {+ await watcher.ready;++ expect(watcher.paths.toSorted()).toEqual([includedFile, excludedFile].toSorted());+ } finally {+ await watcher.close();+ }+ });++ it('should not emit add events for ignored initial files', async () => {+ const watcher = startWatching({ ignoreInitial: false, ignored: ['**/excluded/**'] });++ try {+ await watcher.ready;++ expect(watcher.paths).toEqual([includedFile]);+ } finally {+ await watcher.close();+ }+ });++ it.each(annoyingExclusionPatterns)('should ignore folders with %s in their name', async (char) => {+ const ignoredFolder = `${char}folder`;+ const ignoredFile = join(tempDir, ignoredFolder, 'photo.jpg');+ await mkdir(join(tempDir, ignoredFolder), { recursive: true });+ await writeFile(ignoredFile, 'ignored');++ const watcher = startWatching({ ignoreInitial: false, ignored: [`**/${ignoredFolder}/**`] });++ try {+ await watcher.ready;++ expect(watcher.paths.toSorted()).toEqual([includedFile, excludedFile].toSorted());+ } finally {+ await watcher.close();+ }+ });+});diff --git a/server/test/medium/specs/services/library.service.spec.ts b/server/test/medium/specs/services/library.service.spec.tsindex 395ffc92ec..42730db2d3 100644--- a/server/test/medium/specs/services/library.service.spec.ts+++ b/server/test/medium/specs/services/library.service.spec.ts@@ -3,11 +3,14 @@ import { existsSync } from 'node:fs';import { copyFile, mkdir, mkdtemp, rm, utimes, writeFile } from 'node:fs/promises';import { tmpdir } from 'node:os';import { dirname, join, resolve } from 'node:path';+import type { SystemConfig } from 'src/dtos/config.dto.js';import { StorageCore } from 'src/cores/storage.core.js';import { AssetStatus, JobName, JobStatus } from 'src/enum.js';import { AssetJobRepository } from 'src/repositories/asset-job.repository.js';import { AssetRepository } from 'src/repositories/asset.repository.js';+import { CronRepository } from 'src/repositories/cron.repository.js';import { CryptoRepository } from 'src/repositories/crypto.repository.js';+import { DatabaseRepository } from 'src/repositories/database.repository.js';import { EventRepository } from 'src/repositories/event.repository.js';import { JobRepository } from 'src/repositories/job.repository.js';import { LibraryRepository } from 'src/repositories/library.repository.js';@@ -15,6 +18,7 @@ import { LoggingRepository } from 'src/repositories/logging.repository.js';import { StorageRepository } from 'src/repositories/storage.repository.js';import { DB } from 'src/schema/index.js';import { LibraryService } from 'src/services/library.service.js';+import { systemConfigStub } from 'test/fixtures/system-config.stub.js';import { MediumTestContext, testAssetsDir } from 'test/medium.factory.js';import { newUuid } from 'test/small.factory.js';import { getKyselyDB } from 'test/utils.js';@@ -45,7 +49,7 @@ class LibraryTestContext extends MediumTestContext<typeof LibraryService> {super(LibraryService, {database,real: [AssetRepository, AssetJobRepository, CryptoRepository, LibraryRepository, StorageRepository],- mock: [EventRepository, JobRepository, LoggingRepository],+ mock: [CronRepository, DatabaseRepository, EventRepository, JobRepository, LoggingRepository],});const jobs = this.getMock(JobRepository);@@ -518,6 +522,32 @@ describe(LibraryService.name, () => {});});+ describe('watch', () => {+ it('should pass exclusion patterns to the library watcher', async () => {+ const { sut, ctx } = setup();+ const library = await ctx.createLibrary({+ importPaths: [importRoot],+ exclusionPatterns: ['**/excluded/**'],+ });+ const storage = ctx.get(StorageRepository);+ const watch = vitest.spyOn(storage, 'watch');++ ctx.getMock(DatabaseRepository).tryLock.mockResolvedValue(true);++ try {+ await sut.onConfigInit({ newConfig: systemConfigStub.libraryWatchEnabled as SystemConfig });++ expect(watch).toHaveBeenCalledWith(+ library.importPaths,+ expect.objectContaining({ ignored: library.exclusionPatterns }),+ expect.anything(),+ );+ } finally {+ await sut.onShutdown();+ }+ });+ });+describe('handleQueueSyncAssets', () => {it('should set an asset offline if its file is not in any import path', async () => {const { sut, ctx } = setup();