Skip to content

Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.

fix(backend:files): make highlight context Unicode-aware

johaven6e574c3da1ca
diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts
index dece6008..0d180ed2 100644
--- a/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts
+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts
@@ -427,6 +427,29 @@ describe(FilesContentStoreMySQL.name, () => {
expect(res[0].content).toBeUndefined()
expect(res[0].matches).toEqual(['Le dossier <mark>euro-office</mark> est prêt.'])
})
+
+ it('should keep Unicode words in the highlighted match context', async () => {
+ db.execute.mockResolvedValueOnce([[{ id: 5, sourceIndex: 'files_content_u_1', score: 1 }]])
+ db.execute.mockResolvedValueOnce([
+ [
+ {
+ id: 5,
+ sourceIndex: 'files_content_u_1',
+ path: '/docs',
+ name: 'resume.txt',
+ mime: 'text/plain',
+ mtime: 1730000000004,
+ content: 'Le résumé final mentionne budget.'
+ }
+ ]
+ ])
+
+ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'budget', 10)
+
+ expect(db.execute).toHaveBeenCalledTimes(2)
+ expect(res[0].content).toBeUndefined()
+ expect(res[0].matches).toEqual(['Le résumé final mentionne <mark>budget</mark>.'])
+ })
})
describe('cleanIndexes', () => {
diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
index 61c6d767..8b40a114 100644
--- a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts
@@ -10,6 +10,8 @@ import { genTermsPattern, likeSearchTermStartPattern, MaxSortedList, parseSearch
type SearchCandidate = Pick<FileContent, 'id' | 'score'> & { sourceIndex: string }
type SearchRecord = FileContent & { sourceIndex: string }
+const HIGHLIGHT_CONTEXT_WORD = '(?<![\\p{L}\\p{N}])[\\p{L}\\p{N}]+(?![\\p{L}\\p{N}])'
+const HIGHLIGHT_CONTEXT_SEPARATOR = '[^\\p{L}\\p{N}]'
@Injectable()
export class FilesContentStoreMySQL implements FilesContentStore {
@@ -197,7 +199,10 @@ export class FilesContentStoreMySQL implements FilesContentStore {
})
const termsPattern = `(${genTermsPattern(terms)})`
- const termsRegexp = new RegExp(`(?:\\b\\w+\\b[\\s\\W]{0,4}){0,10}(?:\\b|${likeSearchTermStartPattern()})${termsPattern}(?:\\s*\\S*){0,15}`, 'giu')
+ const termsRegexp = new RegExp(
+ `(?:${HIGHLIGHT_CONTEXT_WORD}${HIGHLIGHT_CONTEXT_SEPARATOR}{0,4}){0,10}(?:\\b|${likeSearchTermStartPattern()})${termsPattern}(?:\\s*\\S*){0,15}`,
+ 'giu'
+ )
const termsHighlightRegexp = new RegExp(termsPattern, 'giu')
for (const r of records) {