Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
feat(backend:files): improve full-text search
johaven7ff6a8b7005c
diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.md b/backend/src/applications/files/adapters/files-content-store-mysql.service.mdnew file mode 100644index 00000000..d6e971db--- /dev/null+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.md@@ -0,0 +1,73 @@+# Content search with MariaDB++`FilesContentStoreMySQL` adapts user input to the Boolean mode of+`MATCH ... AGAINST`, then builds snippets from the selected documents.++## Search syntax++| Input | Behavior |+|--------------------|--------------------------------------------------------------------------|+| `report budget` | Optional terms: at least one of them must match. |+| `+report` | Required term. |+| `-draft` | Excluded term. A search must still contain at least one positive term. |+| `test+` or `test-` | The invalid trailing operator is removed before querying MariaDB. |+| `set*` | Prefix search; `setting` may match. Only a suffix wildcard is supported. |++Sequences of `+` and `-` that do not target a term are removed outside quoted phrases. For example, `test+` becomes `test` and `test+,` becomes+`test,`. Operators placed at the beginning of a term retain their Boolean meaning.++## Compound terms and quoted phrases++An unquoted term containing a separator between multiple tokens is automatically converted into a MariaDB phrase:++| Input | Normalized query |+|----------------------|------------------------|+| `set-variable` | `"set-variable"` |+| `2017-03-05` | `"2017-03-05"` |+| `[email protected]` | `"[email protected]"` |+| `+set-variable` | `+"set-variable"` |++An already quoted phrase, such as `"configure set-variable now"`, remains unchanged. In MariaDB, quotes require a sequence of **tokens**, not a+character-for-character match: `"set-variable"` may therefore match `set variable`. Punctuation (`-`, `@`, `.`, etc.) generally acts as a separator+for the FULLTEXT tokenizer.++## Scripts without FULLTEXT segmentation++If at least one term contains a Han, Hiragana, Katakana, Hangul, Thai, Lao, Khmer, or Myanmar character, the entire query uses `LIKE`:++- required terms are combined with `AND`;+- optional terms are combined with `OR`;+- exclusions use `NOT LIKE`;+- `%`, `_`, and the escape character are escaped.++This fallback enables searches without an `ngram` tokenizer, but it scans content and is more expensive than a FULLTEXT index.++## SQL execution++The search runs in two phases:++1. MariaDB selects only `sourceIndex`, `id`, and `score` from each table, then applies sorting and the limit to the `UNION ALL` result.+2. A second query loads metadata and `LONGTEXT` only for the selected identifiers. The order calculated during the first phase is restored in memory.++The full content is then used to generate snippets, but it is never returned in the search result.++## Snippets and highlighting++- Only positive terms are highlighted with `<mark>`.+- A document returns at most 5 snippets.+- A snippet keeps at most 10 words before and 15 words after the match, with safety limits of 512 and 768 characters.+- Nearby matches are grouped. Snippets containing the most distinct terms, followed by the highest number of occurrences, are prioritized.+- A compound term finds and highlights the separators actually present in the content: `set-variable` may produce+ `<mark>set variable</mark>`.+- Highlighting handles common Latin variants with or without accents, such as `resume` and `résumé`. SQL search also depends on the table collation+ (`utf8mb4_general_ci`).+- With a suffix wildcard, the end-of-word boundary is removed from highlighting.++## Known limitations++- The HTTP query is limited to 512 characters; the result limit defaults to 100 and cannot exceed 100.+- Terms shorter than 2 characters are ignored by the application.+- Stopwords and the minimum indexed token length depend on the MariaDB configuration and storage engine. A term accepted by the application may+ therefore be absent from the index.+- FULLTEXT scores depend on each table's statistics. A score from a user index is not strictly comparable with one from a space or share index.+- A MariaDB phrase compares consecutive tokens; it does not guarantee that the punctuation entered matches the document's punctuation.diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.tsindex a84bf836..5effb56f 100644--- a/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts@@ -18,6 +18,14 @@ describe(FilesContentStoreMySQL.name, () => {return ''}+ const mockSingleSearchRecord = (content: string): void => {+ const id = 1+ const sourceIndex = 'files_content_u_1'+ db.execute+ .mockResolvedValueOnce([[{ id, sourceIndex, score: 1 }]])+ .mockResolvedValueOnce([[{ id, sourceIndex, path: '/docs', name: 'document.txt', mime: 'text/plain', mtime: 1730000000000, content }]])+ }+beforeAll(async () => {db = { execute: vi.fn() }@@ -255,28 +263,29 @@ describe(FilesContentStoreMySQL.name, () => {expect(res).toEqual([])})- it('should use only LIKE for a CJK-only search', async () => {+ it('should remove trailing boolean operators before querying FULLTEXT', async () => {db.execute.mockResolvedValueOnce([[]])- await filesIndexerMySQL.searchRecords(['files_content_u_1'], '中文', 10)+ await filesIndexerMySQL.searchRecords(['files_content_u_1'], '+required optional+ excluded- "C++ guide"', 10)- const query = sqlText(db.execute.mock.calls[0][0])- expect(query).toContain('content LIKE')- expect(query).not.toContain('MATCH (content)')+ const query = dialect.sqlToQuery(db.execute.mock.calls[0][0])+ expect(query.params).toContain('+required optional excluded "C++ guide"')+ expect(query.params).not.toContain('+required optional+ excluded- "C++ guide"')})- it('should use LIKE for non-segmented scripts and MATCH for Devanagari', async () => {- db.execute.mockResolvedValue([[]])+ it.each([+ ['report -set-variable', 'report -"set-variable"'],+ ['2017-03-05', '"2017-03-05"'],+ ['[email protected]', '"[email protected]"'],+ ['"configure set-variable now"', '"configure set-variable now"'],+ ['test+,', 'test,']+ ])('should normalize the FULLTEXT search %s as %s', async (search, normalizedSearch) => {+ db.execute.mockResolvedValueOnce([[]])- await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'ภาษาไทย', 10)- await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'खाते मिलान', 10)+ await filesIndexerMySQL.searchRecords(['files_content_u_1'], search, 10)- const thaiQuery = sqlText(db.execute.mock.calls[0][0])- const devanagariQuery = sqlText(db.execute.mock.calls[1][0])- expect(thaiQuery).toContain('content LIKE')- expect(thaiQuery).not.toContain('MATCH (content)')- expect(devanagariQuery).toContain('MATCH (content)')- expect(devanagariQuery).not.toContain('content LIKE')+ const query = dialect.sqlToQuery(db.execute.mock.calls[0][0])+ expect(query.params.filter((param) => param === normalizedSearch)).toHaveLength(2)})it('should use only LIKE when a mixed search contains CJK', async () => {@@ -289,68 +298,24 @@ describe(FilesContentStoreMySQL.name, () => {expect(query).not.toContain('MATCH (content)')})- it('should translate required, optional, excluded and exact CJK terms', async () => {+ it('should translate required, optional and excluded CJK terms', async () => {db.execute.mockResolvedValue([[]])await filesIndexerMySQL.searchRecords(['files_content_u_1'], '+中文 +文档 -秘密', 10)await filesIndexerMySQL.searchRecords(['files_content_u_1'], '中文 文档', 10)- await filesIndexerMySQL.searchRecords(['files_content_u_1'], '"中文 文档"', 10)const requiredQuery = sqlText(db.execute.mock.calls[0][0])const optionalQuery = sqlText(db.execute.mock.calls[1][0])- const exactPhraseQuery = sqlText(db.execute.mock.calls[2][0])expect(requiredQuery).toContain('WHERE (content LIKE')expect(requiredQuery).toContain("ESCAPE '=' AND content LIKE")expect(requiredQuery).toContain('content NOT LIKE')expect(optionalQuery).toContain("ESCAPE '=' OR content LIKE")- // One LIKE in the score and one in WHERE means the quoted phrase was kept as one term.- expect(exactPhraseQuery.match(/content LIKE/g)).toHaveLength(2)- })-- it('should keep identical ids from different indexes with their score order', async () => {- db.execute.mockResolvedValueOnce([- [- { id: 3, sourceIndex: 'files_content_u_1', score: 10 },- { id: 3, sourceIndex: 'files_content_s_2', score: 5 }- ]- ])- db.execute.mockResolvedValueOnce([- [- {- id: 3,- sourceIndex: 'files_content_s_2',- path: '/shared',- name: 'shared.txt',- mime: 'text/plain',- mtime: 1730000000001,- content: 'foo shared'- },- {- id: 3,- sourceIndex: 'files_content_u_1',- path: '/personal',- name: 'personal.txt',- mime: 'text/plain',- mtime: 1730000000000,- content: 'foo personal'- }- ]- ])-- const res = await filesIndexerMySQL.searchRecords(['files_content_u_1', 'files_content_s_2'], 'foo', 10)-- expect(res).toHaveLength(2)- expect(res.map(({ name, score }) => [name, score])).toEqual([- ['personal.txt', 10],- ['shared.txt', 5]- ])})- it('should search across tables, sort by score, and highlight matches', async () => {- // fabricate records returned by DB. Only first array (rows) is used.+ it('should load final content across indexes, preserve score order, and highlight matches', async () => {const rows = [{- id: 1,+ id: 3,path: '/docs',name: 'alpha.txt',mime: 'text/plain',@@ -359,7 +324,7 @@ describe(FilesContentStoreMySQL.name, () => {score: 10},{- id: 2,+ id: 3,path: '/docs',name: 'beta.txt',mime: 'text/plain',@@ -370,8 +335,8 @@ describe(FilesContentStoreMySQL.name, () => {]db.execute.mockResolvedValueOnce([[- { id: 1, sourceIndex: 'files_content_u_1', score: 10 },- { id: 2, sourceIndex: 'files_content_s_2', score: 5 }+ { id: 3, sourceIndex: 'files_content_u_1', score: 10 },+ { id: 3, sourceIndex: 'files_content_s_2', score: 5 }]])db.execute.mockResolvedValueOnce([@@ -391,30 +356,18 @@ describe(FilesContentStoreMySQL.name, () => {expect(recordsQuery).toContain('path')expect(recordsQuery).toContain('content')expect(res.length).toBe(2)- expect(res.map((record) => record.id)).toEqual([1, 2])- // content must be cleared+ expect(res.map(({ name, score }) => [name, score])).toEqual([+ ['alpha.txt', 10],+ ['beta.txt', 5]+ ])expect(res[0].content).toBeUndefined()expect(Array.isArray(res[0].matches)).toBe(true)expect(res[0].matches!.length).toBeGreaterThan(0)- // highlighted with <mark> tagsexpect(res[0].matches!.join(' ')).toMatch(/<mark>foo<\/mark>|<mark>bar<\/mark>/i)})it('should highlight CJK content returned by the fallback search', async () => {- db.execute.mockResolvedValueOnce([[{ id: 3, sourceIndex: 'files_content_u_1', score: 1 }]])- db.execute.mockResolvedValueOnce([- [- {- id: 3,- sourceIndex: 'files_content_u_1',- path: '/docs',- name: '日本語.txt',- mime: 'text/plain',- mtime: 1730000000002,- content: 'これは日本語の文書です。'- }- ]- ])+ mockSingleSearchRecord('これは日本語の文書です。')const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], '日本語', 10)@@ -423,50 +376,73 @@ describe(FilesContentStoreMySQL.name, () => {expect(res[0].matches).toEqual(['<mark>日本語</mark>の文書です。'])})- it('should highlight hyphenated full-text matches', async () => {- db.execute.mockResolvedValueOnce([[{ id: 4, sourceIndex: 'files_content_u_1', score: 1 }]])- db.execute.mockResolvedValueOnce([- [- {- id: 4,- sourceIndex: 'files_content_u_1',- path: '/docs',- name: 'euro-office.txt',- mime: 'text/plain',- mtime: 1730000000003,- content: 'Le dossier euro-office est prêt.'- }- ]- ])+ it('should highlight a tokenized compound before its shorter alternative', async () => {+ mockSingleSearchRecord('The command can set variable values.')- const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'euro-office', 10)+ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'set set-variable', 10)- expect(db.execute).toHaveBeenCalledTimes(2)- expect(res[0].content).toBeUndefined()- expect(res[0].matches).toEqual(['Le dossier <mark>euro-office</mark> est prêt.'])+ expect(res[0].matches).toEqual(['The command can <mark>set variable</mark> values.'])})- it('should keep Unicode words in the highlighted match context', async () => {- db.execute.mockResolvedValueOnce([[{ id: 5, sourceIndex: 'files_content_u_1', score: 1 }]])- db.execute.mockResolvedValueOnce([- [- {- id: 5,- sourceIndex: 'files_content_u_1',- path: '/docs',- name: 'resume.txt',- mime: 'text/plain',- mtime: 1730000000004,- content: 'Le résumé final mentionne budget.'- }- ]- ])+ it('should bound the highlighted context around a match', async () => {+ const before = Array.from({ length: 12 }, (_, index) => `before${index}`).join(' ')+ const after = Array.from({ length: 17 }, (_, index) => `after${index}`).join(' ')+ mockSingleSearchRecord(`${before} target ${after}.`)- const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'budget', 10)+ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'target', 10)+ const [match] = res[0].matches- expect(db.execute).toHaveBeenCalledTimes(2)- expect(res[0].content).toBeUndefined()- expect(res[0].matches).toEqual(['Le résumé final mentionne <mark>budget</mark>.'])+ expect(match).toMatch(/^before2 /)+ expect(match).toContain('<mark>target</mark>')+ expect(match).toContain('after14')+ expect(match).not.toContain('after15')+ })++ it('should highlight every occurrence inside one context', async () => {+ mockSingleSearchRecord('target target target')++ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'target', 10)++ expect(res[0].matches).toEqual(['<mark>target</mark> <mark>target</mark> <mark>target</mark>'])+ })++ it('should highlight the FULLTEXT token when trailing operators precede punctuation', async () => {+ mockSingleSearchRecord('The test is available.')++ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'test+,', 10)++ expect(res[0].matches).toEqual(['The <mark>test</mark> is available.'])+ })++ it('should highlight an unaccented match from an accented search', async () => {+ mockSingleSearchRecord('Le resume final est disponible.')++ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'résumé', 10)++ expect(res[0].matches).toEqual(['Le <mark>resume</mark> final est disponible.'])+ })++ it('should highlight a Unicode term containing combining marks', async () => {+ mockSingleSearchRecord('यह खाते का विवरण है।')++ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'खाते', 10)++ expect(res[0].matches[0]).toContain('<mark>खाते</mark>')+ })++ it.each([+ ['set', '<mark>set</mark> value', '<mark>set</mark>ting'],+ ['set*', '<mark>set</mark>ting', null]+ ] as const)('should respect the FULLTEXT wildcard boundary for %s', async (search, highlightedTerm, rejectedHighlight) => {+ mockSingleSearchRecord('A setting followed by a set value.')++ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], search, 10)+ const matches = res[0].matches.join(' ')++ expect(matches).toContain(highlightedTerm)+ if (rejectedHighlight) {+ expect(matches).not.toContain(rejectedHighlight)+ }})})diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.tsindex ba710a23..240a1ae6 100644--- a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts@@ -4,19 +4,42 @@ import { MySqlQueryResult } from 'drizzle-orm/mysql2'import { DB_TOKEN_PROVIDER } from '../../../infrastructure/database/constants'import { DBSchema } from '../../../infrastructure/database/interfaces/database.interface'import { FILE_REPOSITORY } from '../constants/operations'+import type { FilesSearchTerm } from '../interfaces/files-search-query.interface'import { FilesContentStore } from '../models/files-content-store'import { FileContent, FileContentRecordMetadata, FileContentRecordMetadataMap } from '../schemas/file-content.interface'import { createTableFilesContent, FILES_CONTENT_TABLE_PREFIX } from '../schemas/files-content.schema'-import { genTermsPattern, likeSearchTermStartPattern, MaxSortedList, parseSearchTerms, SearchTerm } from '../utils/files-search'+import { genTermsPattern, likeSearchTermStartPattern, MaxSortedList, parseFilesSearchQuery } from '../utils/files-search'type SearchCandidate = Pick<FileContent, 'id' | 'score'> & { sourceIndex: string }type SearchRecord = FileContent & { sourceIndex: string }-const HIGHLIGHT_CONTEXT_WORD = '(?<![\\p{L}\\p{N}])[\\p{L}\\p{N}]+(?![\\p{L}\\p{N}])'-const HIGHLIGHT_CONTEXT_SEPARATOR = '[^\\p{L}\\p{N}]'+interface HighlightMatch {+ index: number+ value: string+}+interface HighlightContext {+ start: number+ end: number+ matches: HighlightMatch[]+}+const HIGHLIGHT_WORD_CHAR = '[\\p{L}\\p{M}\\p{N}]'+const HIGHLIGHT_TERM_START = `(?<!${HIGHLIGHT_WORD_CHAR})`+const HIGHLIGHT_TERM_END = `(?!${HIGHLIGHT_WORD_CHAR})`+const HIGHLIGHT_CONTEXT_SEPARATOR = '[^\\p{L}\\p{M}\\p{N}]'+const HIGHLIGHT_UNICODE_WORD_CHARACTER_PATTERN = /[\p{L}\p{M}\p{N}]/uy+const HIGHLIGHT_CONTEXT_WORDS_BEFORE = 10+const HIGHLIGHT_CONTEXT_WORDS_AFTER = 15+const HIGHLIGHT_CONTEXT_CHARS_BEFORE = 512+const HIGHLIGHT_CONTEXT_CHARS_AFTER = 768const FILES_CONTENT_TABLE_PATTERN = new RegExp(`^${FILES_CONTENT_TABLE_PREFIX}(?:${Object.values(FILE_REPOSITORY).join('|')})_[0-9]+$`)@Injectable()export class FilesContentStoreMySQL implements FilesContentStore {+ private static readonly INVALID_TRAILING_FULL_TEXT_OPERATORS = new Set(['+', '-'])+ private static readonly FULL_TEXT_WHITESPACE_PATTERN = /\s/u+ private static readonly FULL_TEXT_TERM_CHARACTER_PATTERN = /[\p{L}\p{M}\p{N}]/u+ private static readonly FULL_TEXT_COMPOUND_TERM_PATTERN =+ /(^|[\s(])([+-]?)([\p{L}\p{M}\p{N}]+(?:[^\p{L}\p{M}\p{N}\s"]+[\p{L}\p{M}\p{N}]+)+)(?=[\s),.!?;:]|$)/gu+ private static readonly FULL_TEXT_TERM_TOKEN_PATTERN = /[\p{L}\p{M}\p{N}]+/guprivate readonly logger = new Logger(FilesContentStoreMySQL.name)constructor(@Inject(DB_TOKEN_PROVIDER) private readonly db: DBSchema) {}@@ -36,8 +59,8 @@ export class FilesContentStoreMySQL implements FilesContentStore {}async existingIndexes(tableSuffixes: string[]): Promise<string[]> {- const currentTables = await this.indexesList()- return tableSuffixes.map((suffix) => this.getIndexName(suffix)).filter((table) => currentTables.indexOf(table) > -1)+ const currentTables = new Set(await this.indexesList())+ return tableSuffixes.map((suffix) => this.getIndexName(suffix)).filter((table) => currentTables.has(table))}async createIndex(tableName: string): Promise<boolean> {@@ -130,14 +153,11 @@ export class FilesContentStoreMySQL implements FilesContentStore {}async searchRecords(tableNames: string[], search: string, limit: number): Promise<FileContent[]> {- const searchTerms = parseSearchTerms(search)- const positiveTerms = searchTerms.filter(({ operator }) => operator !== 'excluded')- const requiredTerms = searchTerms.filter(({ operator }) => operator === 'required')- const optionalTerms = searchTerms.filter(({ operator }) => operator === 'optional')- const excludedTerms = searchTerms.filter(({ operator }) => operator === 'excluded')+ const normalizedSearch = FilesContentStoreMySQL.normalizeFullTextSearch(search)+ const { terms: searchTerms, positiveTerms, requiredTerms, optionalTerms, excludedTerms } = parseFilesSearchQuery(normalizedSearch)const terms = positiveTerms.map(({ regexpValue }) => regexpValue)const useLikeSearch = searchTerms.some(({ requiresLike }) => requiresLike)- this.logger.verbose({ tag: this.searchRecords.name, msg: `convert ${search} -> ${JSON.stringify(terms)}` })+ this.logger.verbose({ tag: this.searchRecords.name, msg: `convert ${search} -> ${normalizedSearch} -> ${JSON.stringify(terms)}` })if (!terms.length) {return []}@@ -151,9 +171,9 @@ export class FilesContentStoreMySQL implements FilesContentStore {const optionalMatch = createContentMatch(optionalTerms, ' OR ')const positiveMatch = requiredMatch || optionalMatchconst excludedMatch = createContentMatch(excludedTerms, ' AND ', true)- const score = positiveTerms.reduce<SQL>(+ const score = optionalTerms.reduce<SQL>((value, term) => sql`${value} + IF(content LIKE ${toLikePattern(term.rawValue)} ESCAPE '=', 1, 0)`,- sql`0`+ sql`${requiredTerms.length}`)return sql`(SELECT ${tableName} as sourceIndex, id, ${score} as scoreFROM ${sql.identifier(tableName)}@@ -163,7 +183,7 @@ export class FilesContentStoreMySQL implements FilesContentStore {LIMIT ${limit})`}- const fullTextMatch = sql`MATCH (content) AGAINST ( ${search} IN BOOLEAN MODE )`+ const fullTextMatch = sql`MATCH (content) AGAINST ( ${normalizedSearch} IN BOOLEAN MODE )`return sql`(SELECT ${tableName} as sourceIndex, id, ${fullTextMatch} as scoreFROM ${sql.identifier(tableName)}WHERE ${fullTextMatch}@@ -175,14 +195,13 @@ export class FilesContentStoreMySQL implements FilesContentStore {.append(sql` ORDER BY score DESC LIMIT ${limit}`)const [candidateRecords]: SearchCandidate[][] = (await this.db.execute(q)) as MySqlQueryResult- const selectedCandidates = candidateRecords.slice(0, limit)- if (!selectedCandidates.length) {+ if (!candidateRecords.length) {return []}// Load LONGTEXT only for the final candidates to avoid carrying it through UNION and ORDER BY.const idsByIndex = new Map<string, number[]>()- for (const candidate of selectedCandidates) {+ for (const candidate of candidateRecords) {const ids = idsByIndex.get(candidate.sourceIndex) || []ids.push(candidate.id)idsByIndex.set(candidate.sourceIndex, ids)@@ -201,28 +220,16 @@ export class FilesContentStoreMySQL implements FilesContentStore {)const [loadedRecords]: SearchRecord[][] = (await this.db.execute(recordsQuery)) as MySqlQueryResultconst recordsByKey = new Map(loadedRecords.map((record) => [`${record.sourceIndex}:${record.id}`, record]))- const records = selectedCandidates.flatMap((candidate) => {+ const records = candidateRecords.flatMap((candidate) => {const record = recordsByKey.get(`${candidate.sourceIndex}:${candidate.id}`)return record ? [{ ...record, score: candidate.score }] : []})- const termsPattern = `(${genTermsPattern(terms)})`- const termsRegexp = new RegExp(- `(?:${HIGHLIGHT_CONTEXT_WORD}${HIGHLIGHT_CONTEXT_SEPARATOR}{0,4}){0,10}(?:\\b|${likeSearchTermStartPattern()})${termsPattern}(?:\\s*\\S*){0,15}`,- 'giu'- )-- const termsHighlightRegexp = new RegExp(termsPattern, 'giu')+ const termsHighlightRegexp = new RegExp(`(${FilesContentStoreMySQL.genFullTextHighlightPattern(positiveTerms)})`, 'giu')for (const r of records) {- const maxSortedList = new MaxSortedList(5)- for (const i of r.content.matchAll(termsRegexp)) {- const matches: string[] = i[0].match(termsHighlightRegexp).map((term) => term.toLowerCase())- const nbDifferentWords: number = matches.length === 1 ? 1 : parseFloat(`${new Set(matches).size}.${matches.length}`)- maxSortedList.insert([nbDifferentWords, i[0]])- }+ r.matches = FilesContentStoreMySQL.extractHighlightedMatches(r.content, termsHighlightRegexp)// Do not expose the full indexed content in search results.r.content = undefined- r.matches = maxSortedList.data.map(([_nb, content]) => content.replace(termsHighlightRegexp, '<mark>$1</mark>'))}return records.map(({ sourceIndex: _sourceIndex, ...record }) => record)}@@ -244,6 +251,169 @@ export class FilesContentStoreMySQL implements FilesContentStore {}}+ private static normalizeFullTextSearch(search: string): string {+ const searchWithQuotedCompounds = this.quoteUnquotedFullTextCompoundTerms(search)+ let normalizedSearch = ''+ let quoted = false+ for (let index = 0; index < searchWithQuotedCompounds.length; index++) {+ const character = searchWithQuotedCompounds[index]+ if (character === '"') {+ quoted = !quoted+ normalizedSearch += character+ continue+ }+ if (!quoted && this.INVALID_TRAILING_FULL_TEXT_OPERATORS.has(character)) {+ let operatorEnd = index + 1+ while (this.INVALID_TRAILING_FULL_TEXT_OPERATORS.has(searchWithQuotedCompounds[operatorEnd])) {+ operatorEnd+++ }+ const previousCharacter = searchWithQuotedCompounds[index - 1]+ const isLeadingOperator = index === 0 || previousCharacter === '(' || this.FULL_TEXT_WHITESPACE_PATTERN.test(previousCharacter)+ const nextCodePoint = searchWithQuotedCompounds.codePointAt(operatorEnd)+ const nextCharacter = nextCodePoint === undefined ? undefined : String.fromCodePoint(nextCodePoint)+ const hasValidTarget =+ nextCharacter !== undefined &&+ (this.FULL_TEXT_TERM_CHARACTER_PATTERN.test(nextCharacter) || (isLeadingOperator && (nextCharacter === '(' || nextCharacter === '"')))+ if (!hasValidTarget) {+ index = operatorEnd - 1+ continue+ }+ normalizedSearch += isLeadingOperator ? character : searchWithQuotedCompounds.slice(index, operatorEnd)+ index = operatorEnd - 1+ continue+ }+ normalizedSearch += character+ }+ return normalizedSearch+ }++ private static quoteUnquotedFullTextCompoundTerms(search: string): string {+ let normalizedSearch = ''+ let quoted = false+ let segmentStart = 0+ for (let index = 0; index < search.length; index++) {+ if (search[index] !== '"') continue+ const segment = search.slice(segmentStart, index)+ normalizedSearch += quoted ? segment : segment.replace(this.FULL_TEXT_COMPOUND_TERM_PATTERN, '$1$2"$3"')+ normalizedSearch += '"'+ quoted = !quoted+ segmentStart = index + 1+ }+ const segment = search.slice(segmentStart)+ return normalizedSearch + (quoted ? segment : segment.replace(this.FULL_TEXT_COMPOUND_TERM_PATTERN, '$1$2"$3"'))+ }++ private static genFullTextHighlightPattern(terms: FilesSearchTerm[]): string {+ const termStart = `(?:${HIGHLIGHT_TERM_START}|${likeSearchTermStartPattern()})`+ const patterns = terms.map(({ rawValue, regexpValue, requiresLike, wildcard }) => {+ const tokens = rawValue.match(this.FULL_TEXT_TERM_TOKEN_PATTERN) || []+ const termPattern = tokens.length+ ? tokens.map((token) => genTermsPattern([token])).join(`${HIGHLIGHT_CONTEXT_SEPARATOR}+`)+ : genTermsPattern([regexpValue])+ return `${termStart}${termPattern}${requiresLike || wildcard ? '' : HIGHLIGHT_TERM_END}`+ })+ return [...new Set(patterns)].sort((left, right) => right.length - left.length).join('|')+ }++ private static extractHighlightedMatches(content: string, termsRegexp: RegExp): string[] {+ const maxSortedList = new MaxSortedList<HighlightContext>(5)+ let previousContextEnd = 0+ termsRegexp.lastIndex = 0+ let match = termsRegexp.exec(content)+ while (match !== null) {+ const contextStart = Math.max(this.findHighlightContextStart(content, match.index), previousContextEnd)+ let contextEnd = this.findHighlightContextEnd(content, match.index + match[0].length)+ const matches: HighlightMatch[] = [{ index: match.index, value: match[0] }]+ const distinctTerms = new Set([match[0].toLowerCase()])++ let nextMatch = termsRegexp.exec(content)+ while (nextMatch !== null && nextMatch.index < contextEnd) {+ matches.push({ index: nextMatch.index, value: nextMatch[0] })+ distinctTerms.add(nextMatch[0].toLowerCase())+ contextEnd = Math.max(contextEnd, nextMatch.index + nextMatch[0].length)+ nextMatch = termsRegexp.exec(content)+ }+ const score = distinctTerms.size * 1000 + Math.min(matches.length, 999)+ maxSortedList.insert([score, { start: contextStart, end: contextEnd, matches }])+ previousContextEnd = contextEnd+ match = nextMatch+ }+ termsRegexp.lastIndex = 0+ return maxSortedList.data.map(([_score, context]) => this.renderHighlightedContext(content, context))+ }++ private static renderHighlightedContext(content: string, context: HighlightContext): string {+ const highlightedParts: string[] = []+ let cursor = context.start+ for (const match of context.matches) {+ highlightedParts.push(content.slice(cursor, match.index), '<mark>', match.value, '</mark>')+ cursor = match.index + match.value.length+ }+ highlightedParts.push(content.slice(cursor, context.end))+ return highlightedParts.join('')+ }++ private static findHighlightContextStart(content: string, matchStart: number): number {+ if (matchStart === 0) return 0+ const previousIndex = this.previousCharacterIndex(content, matchStart)+ if (this.isHighlightWordCharacterAt(content, previousIndex)) return matchStart++ const minIndex = Math.max(0, matchStart - HIGHLIGHT_CONTEXT_CHARS_BEFORE)+ let index = matchStart+ let insideWord = false+ let words = 0+ while (index > minIndex) {+ const characterIndex = this.previousCharacterIndex(content, index)+ const isWordCharacter = this.isHighlightWordCharacterAt(content, characterIndex)+ if (!isWordCharacter && insideWord && ++words === HIGHLIGHT_CONTEXT_WORDS_BEFORE) break+ index = characterIndex+ insideWord = isWordCharacter+ }+ return index+ }++ private static findHighlightContextEnd(content: string, matchEnd: number): number {+ const maxIndex = Math.min(content.length, matchEnd + HIGHLIGHT_CONTEXT_CHARS_AFTER)+ let index = matchEnd+ let insideWord = matchEnd > 0 && this.isHighlightWordCharacterAt(content, this.previousCharacterIndex(content, matchEnd))+ let words = 0+ while (index < maxIndex) {+ const characterEnd = this.nextCharacterIndex(content, index)+ const isWordCharacter = this.isHighlightWordCharacterAt(content, index)+ if (isWordCharacter && !insideWord) {+ if (words === HIGHLIGHT_CONTEXT_WORDS_AFTER) break+ words+++ }+ index = characterEnd+ insideWord = isWordCharacter+ }+ return index+ }++ private static previousCharacterIndex(value: string, index: number): number {+ const characterIndex = index - 1+ if (characterIndex === 0) return 0+ const character = value.charCodeAt(characterIndex)+ const previousCharacter = value.charCodeAt(characterIndex - 1)+ return character >= 0xdc00 && character <= 0xdfff && previousCharacter >= 0xd800 && previousCharacter <= 0xdbff+ ? characterIndex - 1+ : characterIndex+ }++ private static nextCharacterIndex(value: string, index: number): number {+ const codePoint = value.codePointAt(index)+ return index + (codePoint !== undefined && codePoint > 0xffff ? 2 : 1)+ }++ private static isHighlightWordCharacterAt(value: string, index: number): boolean {+ const codeUnit = value.charCodeAt(index)+ if (codeUnit <= 0x7f) {+ return (codeUnit >= 0x30 && codeUnit <= 0x39) || (codeUnit >= 0x41 && codeUnit <= 0x5a) || (codeUnit >= 0x61 && codeUnit <= 0x7a)+ }+ HIGHLIGHT_UNICODE_WORD_CHARACTER_PATTERN.lastIndex = index+ return HIGHLIGHT_UNICODE_WORD_CHARACTER_PATTERN.test(value)+ }+private async getIndexes(): Promise<Record<string, string>[]> {return (await this.db.execute(sql`SHOW TABLES LIKE ${`${FILES_CONTENT_TABLE_PREFIX}%`}`))[0] as any}@@ -262,7 +432,7 @@ function toLikePattern(term: string): string {return `%${term.replaceAll('=', '==').replaceAll('%', '=%').replaceAll('_', '=_')}%`}-function createContentMatch(terms: SearchTerm[], separator: ' AND ' | ' OR ', negate = false): SQL | null {+function createContentMatch(terms: FilesSearchTerm[], separator: ' AND ' | ' OR ', negate = false): SQL | null {if (!terms.length) return nullreturn sql.join(terms.map(({ rawValue }) =>diff --git a/backend/src/applications/files/constants/search.ts b/backend/src/applications/files/constants/search.tsindex 28775a30..6ea777da 100644--- a/backend/src/applications/files/constants/search.ts+++ b/backend/src/applications/files/constants/search.ts@@ -1,3 +1,4 @@export const SEARCH_FILES_MIN_LIMIT = 1export const SEARCH_FILES_DEFAULT_LIMIT = 100export const SEARCH_FILES_MAX_LIMIT = 100+export const SEARCH_FILES_MAX_QUERY_LENGTH = 512diff --git a/backend/src/applications/files/dto/file-operations.dto.ts b/backend/src/applications/files/dto/file-operations.dto.tsindex 1fe721fd..9a243d37 100644--- a/backend/src/applications/files/dto/file-operations.dto.ts+++ b/backend/src/applications/files/dto/file-operations.dto.ts@@ -1,9 +1,23 @@import { Transform } from 'class-transformer'-import { ArrayMinSize, IsArray, IsBoolean, IsDefined, IsIn, IsInt, IsNotEmpty, IsOptional, IsString, IsUrl, Max, Min } from 'class-validator'+import {+ ArrayMinSize,+ IsArray,+ IsBoolean,+ IsDefined,+ IsIn,+ IsInt,+ IsNotEmpty,+ IsOptional,+ IsString,+ IsUrl,+ Max,+ MaxLength,+ Min+} from 'class-validator'import { RejectIfMatch } from '../../../common/decorators'import { regExpInvalidFileName } from '../../../common/shared'import { TAR_EXTENSION, ZIP_EXTENSION } from '../constants/compress'-import { SEARCH_FILES_DEFAULT_LIMIT, SEARCH_FILES_MAX_LIMIT, SEARCH_FILES_MIN_LIMIT } from '../constants/search'+import { SEARCH_FILES_DEFAULT_LIMIT, SEARCH_FILES_MAX_LIMIT, SEARCH_FILES_MAX_QUERY_LENGTH, SEARCH_FILES_MIN_LIMIT } from '../constants/search'export class CopyMoveFileDto {@IsNotEmpty()@@ -70,6 +84,7 @@ export class CompressFileDto {export class SearchFilesDto {@IsNotEmpty()@IsString()+ @MaxLength(SEARCH_FILES_MAX_QUERY_LENGTH)@Transform(({ value }) => (value ? value.trim() : null))content: stringdiff --git a/backend/src/applications/files/interfaces/files-search-query.interface.ts b/backend/src/applications/files/interfaces/files-search-query.interface.tsnew file mode 100644index 00000000..9bbbfb4d--- /dev/null+++ b/backend/src/applications/files/interfaces/files-search-query.interface.ts@@ -0,0 +1,18 @@+export type FilesSearchTermOperator = 'required' | 'excluded' | 'optional'++export interface FilesSearchTerm {+ rawValue: string+ regexpValue: string+ operator: FilesSearchTermOperator+ requiresLike: boolean+ quoted: boolean+ wildcard: boolean+}++export interface FilesSearchQuery {+ terms: FilesSearchTerm[]+ positiveTerms: FilesSearchTerm[]+ requiredTerms: FilesSearchTerm[]+ optionalTerms: FilesSearchTerm[]+ excludedTerms: FilesSearchTerm[]+}diff --git a/backend/src/applications/files/services/files-search-manager.service.spec.ts b/backend/src/applications/files/services/files-search-manager.service.spec.tsindex abae101a..8a6c5f21 100644--- a/backend/src/applications/files/services/files-search-manager.service.spec.ts+++ b/backend/src/applications/files/services/files-search-manager.service.spec.ts@@ -162,6 +162,20 @@ describe(FilesSearchManager.name, () => {expect(error.message).toBe('SyntaxError (check special characters)')})+ it('should keep storage error details in logs but not in the HTTP error', async () => {+ const storageError = new Error('Failed query: SELECT private_data')+ const loggerSpy = vi.spyOn((service as any).logger, 'error').mockImplementation(() => undefined)+ filesIndexer.existingIndexes.mockResolvedValueOnce(['user_5'])+ filesIndexer.searchRecords.mockRejectedValueOnce(storageError)++ const error = await (service as any).searchFullText(5, [], [], 'report', 5).catch((e: HttpException) => e)++ expect(loggerSpy).toHaveBeenCalledWith(expect.objectContaining({ err: storageError }))+ expect(error.getStatus()).toBe(HttpStatus.BAD_REQUEST)+ expect(error.message).toBe('Unable to perform full-text search')+ expect(error.message).not.toContain('private_data')+ })+it('should stop filename search when the limit is reached', async () => {filesParser.allPaths.mockResolvedValue([{diff --git a/backend/src/applications/files/services/files-search-manager.service.ts b/backend/src/applications/files/services/files-search-manager.service.tsindex df78f4ca..2a0c4586 100644--- a/backend/src/applications/files/services/files-search-manager.service.ts+++ b/backend/src/applications/files/services/files-search-manager.service.ts@@ -57,12 +57,13 @@ export class FilesSearchManager {try {return await this.filesIndexer.searchRecords(indexNames, search, limit)} catch (e) {- this.logger.error({ tag: this.searchFullText.name, msg: `${JSON.stringify(indexNames)} - ${search} : ${e}` })+ const error = e instanceof Error ? e : new Error(`${e}`)+ this.logger.error({ tag: this.searchFullText.name, msg: `${JSON.stringify(indexNames)} - ${search}`, err: error })let msg: string- if (/Invalid regular expression/.test(e.message)) {+ if (/Invalid regular expression/.test(error.message)) {msg = 'SyntaxError (check special characters)'} else {- msg = e.message+ msg = 'Unable to perform full-text search'}throw new HttpException(msg, HttpStatus.BAD_REQUEST)}diff --git a/backend/src/applications/files/utils/files-search.spec.ts b/backend/src/applications/files/utils/files-search.spec.tsindex 1070c175..27984077 100644--- a/backend/src/applications/files/utils/files-search.spec.ts+++ b/backend/src/applications/files/utils/files-search.spec.ts@@ -5,6 +5,7 @@ import {likeSearchTermStartPattern,MaxSortedList,normalizeSearchLimit,+ parseFilesSearchQuery,parseSearchTerms,requiresLikeSearch} from './files-search'@@ -80,12 +81,9 @@ describe('files search utilities', () => {})describe(genTermsPattern.name, () => {- it('should generate an accent-insensitive alternative pattern', () => {- const regexp = new RegExp(`^(${genTermsPattern(['resume', 'canyon'])})$`, 'iu')-- expect(regexp.test('résumé')).toBe(true)- expect(regexp.test('cañyon')).toBe(true)- expect(regexp.test('other')).toBe(false)+ it('should match accented and unaccented terms in both directions', () => {+ expect(new RegExp(`^(${genTermsPattern(['resume'])})$`, 'iu').test('résumé')).toBe(true)+ expect(new RegExp(`^(${genTermsPattern(['résumé'])})$`, 'iu').test('resume')).toBe(true)})it('should generate a Unicode regular expression compatible pattern with hyphenated terms', () => {@@ -152,32 +150,45 @@ describe('files search utilities', () => {describe(parseSearchTerms.name, () => {it('should classify boolean search terms and exact phrases', () => {expect(parseSearchTerms('+中文 -秘密 文档 "全文 搜索"')).toEqual([- { rawValue: '中文', regexpValue: '中文', operator: 'required', requiresLike: true },- { rawValue: '秘密', regexpValue: '秘密', operator: 'excluded', requiresLike: true },- { rawValue: '文档', regexpValue: '文档', operator: 'optional', requiresLike: true },- { rawValue: '全文 搜索', regexpValue: '全文 搜索', operator: 'optional', requiresLike: true }+ { rawValue: '中文', regexpValue: '中文', operator: 'required', requiresLike: true, quoted: false, wildcard: false },+ { rawValue: '秘密', regexpValue: '秘密', operator: 'excluded', requiresLike: true, quoted: false, wildcard: false },+ { rawValue: '文档', regexpValue: '文档', operator: 'optional', requiresLike: true, quoted: false, wildcard: false },+ { rawValue: '全文 搜索', regexpValue: '全文 搜索', operator: 'optional', requiresLike: true, quoted: true, wildcard: false }])})it('should remove nested modifiers and trailing wildcards', () => {expect(parseSearchTerms('++report file*')).toEqual([- { rawValue: 'report', regexpValue: 'report', operator: 'required', requiresLike: false },- { rawValue: 'file', regexpValue: 'file', operator: 'optional', requiresLike: false }+ { rawValue: 'report', regexpValue: 'report', operator: 'required', requiresLike: false, quoted: false, wildcard: false },+ { rawValue: 'file', regexpValue: 'file', operator: 'optional', requiresLike: false, quoted: false, wildcard: true }])})it('should ignore terms below the minimum length', () => {- expect(parseSearchTerms('+a valid')).toEqual([{ rawValue: 'valid', regexpValue: 'valid', operator: 'optional', requiresLike: false }])+ expect(parseSearchTerms('+a valid')).toEqual([+ { rawValue: 'valid', regexpValue: 'valid', operator: 'optional', requiresLike: false, quoted: false, wildcard: false }+ ])})it('should expose raw and regular expression values', () => {expect(parseSearchTerms('file.txt euro-office')).toEqual([- { rawValue: 'file.txt', regexpValue: 'file\\.txt', operator: 'optional', requiresLike: false },- { rawValue: 'euro-office', regexpValue: 'euro-office', operator: 'optional', requiresLike: false }+ { rawValue: 'file.txt', regexpValue: 'file\\.txt', operator: 'optional', requiresLike: false, quoted: false, wildcard: false },+ { rawValue: 'euro-office', regexpValue: 'euro-office', operator: 'optional', requiresLike: false, quoted: false, wildcard: false }])})})+ describe(parseFilesSearchQuery.name, () => {+ it('should group generic search terms by operator', () => {+ const query = parseFilesSearchQuery('+required optional -excluded')++ expect(query.positiveTerms.map(({ rawValue }) => rawValue)).toEqual(['required', 'optional'])+ expect(query.requiredTerms.map(({ rawValue }) => rawValue)).toEqual(['required'])+ expect(query.optionalTerms.map(({ rawValue }) => rawValue)).toEqual(['optional'])+ expect(query.excludedTerms.map(({ rawValue }) => rawValue)).toEqual(['excluded'])+ })+ })+describe(likeSearchTermStartPattern.name, () => {it('should match only before a character requiring the LIKE fallback', () => {const regexp = new RegExp(`${likeSearchTermStartPattern()}.`, 'u')diff --git a/backend/src/applications/files/utils/files-search.ts b/backend/src/applications/files/utils/files-search.tsindex ba6644cc..84a368c9 100644--- a/backend/src/applications/files/utils/files-search.ts+++ b/backend/src/applications/files/utils/files-search.ts@@ -1,48 +1,36 @@import { regexpEscape } from '../../../common/functions'import { MIN_CHARS_TO_SEARCH } from '../constants/indexing'import { SEARCH_FILES_DEFAULT_LIMIT, SEARCH_FILES_MAX_LIMIT, SEARCH_FILES_MIN_LIMIT } from '../constants/search'+import type { FilesSearchQuery, FilesSearchTerm } from '../interfaces/files-search-query.interface'-const regexMatchSearchBoolean = new RegExp(`([+-]?)(?:"([^"]+)"|(\\S+))`)-const regexMatchesSearchBoolean = new RegExp(regexMatchSearchBoolean.source, 'g')-const booleanOperators = new Set(['+', '-', '<', '>', '~', '*'])-const UNICODE_WORD_CHAR = '[\\p{L}\\p{N}]'+const SEARCH_TERMS_PATTERN = /([+-]?)(?:"([^"]+)"|(\S+))/g+const LEADING_BOOLEAN_OPERATORS = new Set(['+', '-', '<', '>', '~', '*'])+const UNICODE_WORD_CHAR = '[\\p{L}\\p{M}\\p{N}]'const LIKE_SEARCH_CHAR ='[\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Hangul}\\p{Script=Thai}\\p{Script=Lao}\\p{Script=Khmer}\\p{Script=Myanmar}]'-const regexRequiresLikeSearch = new RegExp(LIKE_SEARCH_CHAR, 'u')-const accentToBaseMap = new Map<string, string>([- ['a', '[aàáâä]'],- ['e', '[eèéêë]'],- ['i', '[iìíîï]'],- ['o', '[oòóôö]'],- ['u', '[uùúûü]'],- ['c', '[cç]'],- ['n', '[nñ]'],- ['s', '[sš]'],- ['z', '[zž]'],- ['y', '[yýÿ]']-])--export interface SearchTerm {- rawValue: string- regexpValue: string- operator: 'required' | 'excluded' | 'optional'- requiresLike: boolean-}+const REQUIRES_LIKE_SEARCH_PATTERN = new RegExp(LIKE_SEARCH_CHAR, 'u')+const ACCENT_INSENSITIVE_CHARACTER_GROUPS = ['aàáâä', 'eèéêë', 'iìíîï', 'oòóôö', 'uùúûü', 'cç', 'nñ', 'sš', 'zž', 'yýÿ']+const ACCENT_INSENSITIVE_PATTERN_BY_CHARACTER = new Map<string, string>(+ ACCENT_INSENSITIVE_CHARACTER_GROUPS.flatMap((characters) => {+ const pattern = `[${characters}]`+ return [...characters].map((character) => [character, pattern] as const)+ })+)-export class MaxSortedList {- public data: [number, string][] = []+export class MaxSortedList<T = string> {+ public data: [number, T][] = []public nbItems: numberconstructor(nbItems: number) {this.nbItems = nbItems}- insert(item: [number, string]) {+ insert(item: [number, T]) {if (this.data.length === 0) {this.data.push(item)return}- // if score is smaller or the score already stored for another string ignore it and keep the first matches.+ // If the score is smaller or already stored for another item, keep the first matches.if (this.data.length === this.nbItems && (item[0] < this.data[this.data.length - 1][0] || this.data.some(([num]) => num === item[0]))) {return}@@ -86,33 +74,47 @@ export function genRegexPositiveAndNegativeTerms(search: string): RegExp {return new RegExp(`^${p}(?!.*(${n})).*$`, 'iu')}-export function requiresLikeSearch(input: string): boolean {- return regexRequiresLikeSearch.test(input)+export function parseFilesSearchQuery(search: string): FilesSearchQuery {+ const terms = parseSearchTerms(search)+ return {+ terms,+ positiveTerms: terms.filter(({ operator }) => operator !== 'excluded'),+ requiredTerms: terms.filter(({ operator }) => operator === 'required'),+ optionalTerms: terms.filter(({ operator }) => operator === 'optional'),+ excludedTerms: terms.filter(({ operator }) => operator === 'excluded')+ }}-export function parseSearchTerms(search: string): SearchTerm[] {- return (search.match(regexMatchesSearchBoolean) || []).flatMap((match: string) => {- const [, operator, quoted, unquoted] = match.match(regexMatchSearchBoolean)- let rawValue = (quoted || unquoted).trim()- while (booleanOperators.has(rawValue[0])) {+export function parseSearchTerms(search: string): FilesSearchTerm[] {+ const terms: FilesSearchTerm[] = []+ for (const [, operator, quotedValue, unquotedValue] of search.matchAll(SEARCH_TERMS_PATTERN)) {+ const quoted = quotedValue !== undefined+ let rawValue = (quotedValue || unquotedValue).trim()+ while (LEADING_BOOLEAN_OPERATORS.has(rawValue[0])) {rawValue = rawValue.substring(1)}- if (rawValue[rawValue.length - 1] === '*') {+ const wildcard = rawValue.endsWith('*')+ if (wildcard) {rawValue = rawValue.substring(0, rawValue.length - 1)}if (rawValue.length < MIN_CHARS_TO_SEARCH) {- return []+ continue}- const searchOperator: SearchTerm['operator'] = operator === '+' ? 'required' : operator === '-' ? 'excluded' : 'optional'- return [- {- rawValue,- regexpValue: escapeSearchTermRegexp(rawValue),- operator: searchOperator,- requiresLike: requiresLikeSearch(rawValue)- }- ]- })+ const searchOperator: FilesSearchTerm['operator'] = operator === '+' ? 'required' : operator === '-' ? 'excluded' : 'optional'+ terms.push({+ rawValue,+ regexpValue: escapeSearchTermRegexp(rawValue),+ operator: searchOperator,+ requiresLike: requiresLikeSearch(rawValue),+ quoted,+ wildcard+ })+ }+ return terms+}++export function requiresLikeSearch(input: string): boolean {+ return REQUIRES_LIKE_SEARCH_PATTERN.test(input)}export function likeSearchTermStartPattern(): string {@@ -132,11 +134,11 @@ function termBoundaryPattern(term: string, endBoundary = false): string {}function genAccentInsensitiveRegexpPattern(input: string): string {- /* Allow to catch all terms with accents or not */- return input- .split('')- .map((char: string) => accentToBaseMap.get(char) || char)- .join('')+ let pattern = ''+ for (const character of input) {+ pattern += ACCENT_INSENSITIVE_PATTERN_BY_CHARACTER.get(character.toLowerCase()) || character+ }+ return pattern}function escapeSearchTermRegexp(input: string): string {