Skip to content

High performance self-hosted photo and video management solution.

fix(server): compare integrity paths independently of unicode normalization (#31102)

Aditya Raj Singh351ddff320e3
diff --git a/server/src/services/integrity.service.spec.ts b/server/src/services/integrity.service.spec.ts
index 1144b1d0fa..ef8a2abef7 100644
--- a/server/src/services/integrity.service.spec.ts
+++ b/server/src/services/integrity.service.spec.ts
@@ -13,6 +13,35 @@ describe(IntegrityService.name, () => {
expect(sut).toBeDefined();
});
+ describe('handleUntrackedFiles', () => {
+ const decomposedPath = '/data/upload/upload/user-id/Cafe\u{301}.jpg';
+ const composedPath = '/data/upload/upload/user-id/Caf\u{E9}.jpg';
+
+ beforeEach(() => {
+ mocks.integrityReport.getAssetPathsByPaths.mockResolvedValue([]);
+ mocks.integrityReport.getPersonThumbnailPathsByPaths.mockResolvedValue([]);
+ });
+
+ it.each([
+ { scannedPath: decomposedPath, storedPath: composedPath },
+ { scannedPath: composedPath, storedPath: decomposedPath },
+ ])(
+ 'should not report a tracked asset whose path differs only by unicode normalization',
+ async ({ scannedPath, storedPath }) => {
+ mocks.integrityReport.getAssetPathsByPaths.mockResolvedValue([
+ { originalPath: storedPath, encodedVideoPath: null },
+ ]);
+
+ await sut.handleUntrackedFiles({ type: 'asset', paths: [scannedPath] });
+
+ expect(mocks.integrityReport.getAssetPathsByPaths).toHaveBeenCalledWith(
+ expect.arrayContaining([decomposedPath, composedPath]),
+ );
+ expect(mocks.integrityReport.create).not.toHaveBeenCalled();
+ },
+ );
+ });
+
describe('handleUntrackedRefresh', () => {
beforeEach(() => {
mocks.integrityReport.getTrackedPaths.mockResolvedValue([]);
diff --git a/server/src/services/integrity.service.ts b/server/src/services/integrity.service.ts
index 4ead01aac7..5013620564 100644
--- a/server/src/services/integrity.service.ts
+++ b/server/src/services/integrity.service.ts
@@ -275,38 +275,42 @@ export class IntegrityService extends BaseService {
async handleUntrackedFiles({ type, paths }: IIntegrityUntrackedFilesJob): Promise<JobStatus> {
this.logger.log(`Processing batch of ${paths.length} files to check if they are untracked.`);
- const untrackedFiles = new Set<string>(paths);
+ const lookupPaths = [...new Set(paths.flatMap((path) => [path, path.normalize('NFC'), path.normalize('NFD')]))];
+
+ const trackedPaths = new Set<string>();
if (type === 'asset') {
- const assets = await this.integrityRepository.getAssetPathsByPaths(paths);
+ const assets = await this.integrityRepository.getAssetPathsByPaths(lookupPaths);
for (const { originalPath, encodedVideoPath } of assets) {
- untrackedFiles.delete(originalPath);
+ trackedPaths.add(originalPath.normalize('NFC'));
if (encodedVideoPath) {
- untrackedFiles.delete(encodedVideoPath);
+ trackedPaths.add(encodedVideoPath.normalize('NFC'));
}
}
} else {
- const assets = await this.integrityRepository.getAssetFilePathsByPaths(paths);
+ const assets = await this.integrityRepository.getAssetFilePathsByPaths(lookupPaths);
for (const { path } of assets) {
- untrackedFiles.delete(path);
+ trackedPaths.add(path.normalize('NFC'));
}
}
- const personThumbnailPaths = await this.integrityRepository.getPersonThumbnailPathsByPaths(paths);
+ const personThumbnailPaths = await this.integrityRepository.getPersonThumbnailPathsByPaths(lookupPaths);
for (const { thumbnailPath } of personThumbnailPaths) {
- untrackedFiles.delete(thumbnailPath);
+ trackedPaths.add(thumbnailPath.normalize('NFC'));
}
- if (untrackedFiles.size > 0) {
+ const untrackedFiles = [...new Set(paths)].filter((path) => !trackedPaths.has(path.normalize('NFC')));
+
+ if (untrackedFiles.length > 0) {
await this.integrityRepository.create(
- [...untrackedFiles].map((path) => ({
+ untrackedFiles.map((path) => ({
type: IntegrityReport.UntrackedFile,
path,
})),
);
}
- this.logger.log(`Processed ${paths.length} and found ${untrackedFiles.size} untracked file(s).`);
+ this.logger.log(`Processed ${paths.length} and found ${untrackedFiles.length} untracked file(s).`);
return JobStatus.Success;
}