High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
fix(server): ensure new exclusion patterns work (#12102)
Jonathan Jogenforsbab5ad7ebd34
diff --git a/e2e/src/api/specs/library.e2e-spec.ts b/e2e/src/api/specs/library.e2e-spec.tsindex 013e1364ca..ec42cbe4fa 100644--- a/e2e/src/api/specs/library.e2e-spec.ts+++ b/e2e/src/api/specs/library.e2e-spec.ts@@ -353,7 +353,7 @@ describe('/libraries', () => {expect(assets.count).toBe(2);- utils.createImageFile(`${testAssetDir}/temp/directoryA/assetB.png`);+ utils.createImageFile(`${testAssetDir}/temp/directoryA/assetC.png`);await scan(admin.accessToken, library.id);await utils.waitForWebsocketEvent({ event: 'assetUpload', total: 3 });@@ -361,11 +361,11 @@ describe('/libraries', () => {const { assets: newAssets } = await utils.metadataSearch(admin.accessToken, { libraryId: library.id });expect(newAssets.count).toBe(3);- utils.removeImageFile(`${testAssetDir}/temp/directoryA/assetB.png`);+ utils.removeImageFile(`${testAssetDir}/temp/directoryA/assetC.png`);});it('should offline a file missing from disk', async () => {- utils.createImageFile(`${testAssetDir}/temp/directoryA/assetB.png`);+ utils.createImageFile(`${testAssetDir}/temp/directoryA/assetC.png`);const library = await utils.createLibrary(admin.accessToken, {ownerId: admin.userId,importPaths: [`${testAssetDirInternal}/temp`],@@ -374,26 +374,28 @@ describe('/libraries', () => {await scan(admin.accessToken, library.id);await utils.waitForQueueFinish(admin.accessToken, 'library');- utils.removeImageFile(`${testAssetDir}/temp/directoryA/assetB.png`);+ const { assets } = await utils.metadataSearch(admin.accessToken, { libraryId: library.id });+ expect(assets.count).toBe(3);++ utils.removeImageFile(`${testAssetDir}/temp/directoryA/assetC.png`);await scan(admin.accessToken, library.id);await utils.waitForQueueFinish(admin.accessToken, 'library');- const { assets } = await utils.metadataSearch(admin.accessToken, { libraryId: library.id });+ const { assets: newAssets } = await utils.metadataSearch(admin.accessToken, { libraryId: library.id });+ expect(newAssets.count).toBe(3);- expect(assets.items).toEqual(+ expect(newAssets.items).toEqual(expect.arrayContaining([expect.objectContaining({isOffline: true,- originalFileName: 'assetB.png',+ originalFileName: 'assetC.png',}),]),);});it('should offline a file outside of import paths', async () => {- utils.createImageFile(`${testAssetDir}/temp/directoryA/assetB.png`);- utils.createImageFile(`${testAssetDir}/temp/directoryB/assetC.png`);const library = await utils.createLibrary(admin.accessToken, {ownerId: admin.userId,importPaths: [`${testAssetDirInternal}/temp`],@@ -416,16 +418,49 @@ describe('/libraries', () => {expect.arrayContaining([expect.objectContaining({isOffline: false,- originalFileName: 'assetB.png',+ originalFileName: 'assetA.png',}),expect.objectContaining({isOffline: true,- originalFileName: 'assetC.png',+ originalFileName: 'assetB.png',}),]),);+ });- utils.removeImageFile(`${testAssetDir}/temp/directoryB/assetC.png`);+ it('should offline a file covered by an exclusion pattern', async () => {+ const library = await utils.createLibrary(admin.accessToken, {+ ownerId: admin.userId,+ importPaths: [`${testAssetDirInternal}/temp`],+ });++ await scan(admin.accessToken, library.id);+ await utils.waitForQueueFinish(admin.accessToken, 'library');++ await request(app)+ .put(`/libraries/${library.id}`)+ .set('Authorization', `Bearer ${admin.accessToken}`)+ .send({ exclusionPatterns: ['**/directoryB/**'] });++ await scan(admin.accessToken, library.id);+ await utils.waitForQueueFinish(admin.accessToken, 'library');++ const { assets } = await utils.metadataSearch(admin.accessToken, { libraryId: library.id });++ expect(assets.count).toBe(2);++ expect(assets.items).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ isOffline: false,+ originalFileName: 'assetA.png',+ }),+ expect.objectContaining({+ isOffline: true,+ originalFileName: 'assetB.png',+ }),+ ]),+ );});it('should not try to delete offline files', async () => {@@ -471,6 +506,8 @@ describe('/libraries', () => {await utils.waitForWebsocketEvent({ event: 'assetDelete', total: 1 });expect(existsSync(`${testAssetDir}/temp/offline1/assetA.png`)).toBe(true);++ utils.removeImageFile(`${testAssetDir}/temp/offline1/assetA.png`);});it('should scan new files', async () => {@@ -482,14 +519,14 @@ describe('/libraries', () => {await scan(admin.accessToken, library.id);await utils.waitForQueueFinish(admin.accessToken, 'library');- utils.createImageFile(`${testAssetDir}/temp/directoryA/assetC.png`);+ utils.createImageFile(`${testAssetDir}/temp/directoryC/assetC.png`);await scan(admin.accessToken, library.id);await utils.waitForQueueFinish(admin.accessToken, 'library');- utils.removeImageFile(`${testAssetDir}/temp/directoryA/assetC.png`);const { assets } = await utils.metadataSearch(admin.accessToken, { libraryId: library.id });+ expect(assets.count).toBe(3);expect(assets.items).toEqual(expect.arrayContaining([expect.objectContaining({@@ -497,6 +534,8 @@ describe('/libraries', () => {}),]),);++ utils.removeImageFile(`${testAssetDir}/temp/directoryC/assetC.png`);});describe('with refreshModifiedFiles=true', () => {diff --git a/server/package-lock.json b/server/package-lock.jsonindex 1ec4fe0fb0..e90256e29b 100644--- a/server/package-lock.json+++ b/server/package-lock.json@@ -51,7 +51,7 @@"nodemailer": "^6.9.13","openid-client": "^5.4.3","pg": "^8.11.3",- "picomatch": "^4.0.0",+ "picomatch": "^4.0.2","react": "^18.3.1","react-email": "^3.0.0","reflect-metadata": "^0.2.0",@@ -11403,6 +11403,7 @@"version": "4.0.2","resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz","integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==",+ "license": "MIT","engines": {"node": ">=12"},diff --git a/server/package.json b/server/package.jsonindex 9b42922278..42552f20b7 100644--- a/server/package.json+++ b/server/package.json@@ -77,7 +77,7 @@"nodemailer": "^6.9.13","openid-client": "^5.4.3","pg": "^8.11.3",- "picomatch": "^4.0.0",+ "picomatch": "^4.0.2","react": "^18.3.1","react-email": "^3.0.0","reflect-metadata": "^0.2.0",diff --git a/server/src/interfaces/job.interface.ts b/server/src/interfaces/job.interface.tsindex fab959936f..b2ac5ec6f1 100644--- a/server/src/interfaces/job.interface.ts+++ b/server/src/interfaces/job.interface.ts@@ -133,6 +133,7 @@ export interface ILibraryFileJob extends IEntityJob {export interface ILibraryOfflineJob extends IEntityJob {importPaths: string[];+ exclusionPatterns: string[];}export interface ILibraryRefreshJob extends IEntityJob {diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.tsindex 9e260e98ef..2d4e1d5776 100644--- a/server/src/services/library.service.spec.ts+++ b/server/src/services/library.service.spec.ts@@ -301,6 +301,7 @@ describe(LibraryService.name, () => {const mockAssetJob: ILibraryOfflineJob = {id: assetStub.external.id,importPaths: ['/'],+ exclusionPatterns: [],};assetMock.getById.mockResolvedValue(null);@@ -314,6 +315,7 @@ describe(LibraryService.name, () => {const mockAssetJob: ILibraryOfflineJob = {id: assetStub.external.id,importPaths: ['/'],+ exclusionPatterns: [],};assetMock.getById.mockResolvedValue(assetStub.offline);@@ -323,10 +325,25 @@ describe(LibraryService.name, () => {expect(assetMock.update).not.toHaveBeenCalled();});- it('should offline assets no longer on disk or matching exclusion pattern', async () => {+ it('should offline assets no longer on disk', async () => {const mockAssetJob: ILibraryOfflineJob = {id: assetStub.external.id,importPaths: ['/'],+ exclusionPatterns: [],+ };++ assetMock.getById.mockResolvedValue(assetStub.external);++ await expect(sut.handleOfflineCheck(mockAssetJob)).resolves.toBe(JobStatus.SUCCESS);++ expect(assetMock.update).toHaveBeenCalledWith({ id: assetStub.external.id, isOffline: true });+ });++ it('should offline assets matching an exclusion pattern', async () => {+ const mockAssetJob: ILibraryOfflineJob = {+ id: assetStub.external.id,+ importPaths: ['/'],+ exclusionPatterns: ['**/user1/**'],};assetMock.getById.mockResolvedValue(assetStub.external);@@ -340,6 +357,7 @@ describe(LibraryService.name, () => {const mockAssetJob: ILibraryOfflineJob = {id: assetStub.external.id,importPaths: ['/data/user2'],+ exclusionPatterns: [],};assetMock.getById.mockResolvedValue(assetStub.external);@@ -354,6 +372,7 @@ describe(LibraryService.name, () => {const mockAssetJob: ILibraryOfflineJob = {id: assetStub.external.id,importPaths: ['/'],+ exclusionPatterns: [],};assetMock.getById.mockResolvedValue(assetStub.external);diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.tsindex 9e31107027..c7f82eddea 100644--- a/server/src/services/library.service.ts+++ b/server/src/services/library.service.ts@@ -556,11 +556,16 @@ export class LibraryService {return JobStatus.SUCCESS;}+ const isExcluded = job.exclusionPatterns.some((pattern) => picomatch.isMatch(asset.originalPath, pattern));+ if (isExcluded) {+ this.logger.debug(`Asset is covered by an exclusion pattern, marking offline: ${asset.originalPath}`);+ await this.assetRepository.update({ id: asset.id, isOffline: true });+ return JobStatus.SUCCESS;+ }+const fileExists = await this.storageRepository.checkFileExists(asset.originalPath, R_OK);if (!fileExists) {- this.logger.debug(- `Asset is no longer found on disk or is covered by exclusion pattern, marking offline: ${asset.originalPath}`,- );+ this.logger.debug(`Asset is no longer found on disk, marking offline: ${asset.originalPath}`);await this.assetRepository.update({ id: asset.id, isOffline: true });return JobStatus.SUCCESS;}