Sync-in · Sovereign platform for file storage, sharing, synchronization, and collaboration.
TypeScript cloud-alternativecollaborationcollaborative-editingdropboxfile-serverfile-sharingfile-storagefile-syncfull-text-searchhomelabnextcloudon-premiseopen-sourceprivacy-firstprivate-cloudrealtimesecureself-hostedstorage-serverwebdav
fix(backend:files): support multilingual full-text search
johaven9a462c501f32
diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.tsindex 41408154..13df0ff4 100644--- a/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.spec.ts@@ -8,6 +8,14 @@ describe(FilesContentStoreMySQL.name, () => {let filesIndexerMySQL: FilesContentStoreMySQLlet db: { execute: Mock }+ const sqlText = (query: any): string => {+ if (typeof query === 'string') return query+ if (Array.isArray(query)) return query.map(sqlText).join('')+ if (Array.isArray(query?.value)) return query.value.join('')+ if (Array.isArray(query?.queryChunks)) return query.queryChunks.map(sqlText).join('')+ return ''+ }+beforeAll(async () => {db = { execute: vi.fn() }@@ -220,9 +228,106 @@ describe(FilesContentStoreMySQL.name, () => {const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'foo', 10)expect(db.execute).toHaveBeenCalledTimes(1)+ const query = sqlText(db.execute.mock.calls[0][0])+ expect(query).toContain('SELECT')+ expect(query).not.toContain('path')+ expect(query).not.toContain('content,')+ expect(query).toContain('MATCH (content)')+ expect(query).not.toContain('content LIKE')expect(res).toEqual([])})+ it('should use only LIKE for a CJK-only search', async () => {+ db.execute.mockResolvedValueOnce([[]])++ await filesIndexerMySQL.searchRecords(['files_content_u_1'], '中文', 10)++ const query = sqlText(db.execute.mock.calls[0][0])+ expect(query).toContain('content LIKE')+ expect(query).not.toContain('MATCH (content)')+ })++ it('should use LIKE for non-segmented scripts and MATCH for Devanagari', async () => {+ db.execute.mockResolvedValue([[]])++ await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'ภาษาไทย', 10)+ await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'खाते मिलान', 10)++ const thaiQuery = sqlText(db.execute.mock.calls[0][0])+ const devanagariQuery = sqlText(db.execute.mock.calls[1][0])+ expect(thaiQuery).toContain('content LIKE')+ expect(thaiQuery).not.toContain('MATCH (content)')+ expect(devanagariQuery).toContain('MATCH (content)')+ expect(devanagariQuery).not.toContain('content LIKE')+ })++ it('should use only LIKE when a mixed search contains CJK', async () => {+ db.execute.mockResolvedValueOnce([[]])++ await filesIndexerMySQL.searchRecords(['files_content_u_1'], 'report 中文', 10)++ const query = sqlText(db.execute.mock.calls[0][0])+ expect(query).toContain('WHERE (content LIKE')+ expect(query).not.toContain('MATCH (content)')+ })++ it('should translate required, optional, excluded and exact CJK terms', async () => {+ db.execute.mockResolvedValue([[]])++ await filesIndexerMySQL.searchRecords(['files_content_u_1'], '+中文 +文档 -秘密', 10)+ await filesIndexerMySQL.searchRecords(['files_content_u_1'], '中文 文档', 10)+ await filesIndexerMySQL.searchRecords(['files_content_u_1'], '"中文 文档"', 10)++ const requiredQuery = sqlText(db.execute.mock.calls[0][0])+ const optionalQuery = sqlText(db.execute.mock.calls[1][0])+ const exactPhraseQuery = sqlText(db.execute.mock.calls[2][0])+ expect(requiredQuery).toContain('WHERE (content LIKE')+ expect(requiredQuery).toContain("ESCAPE '=' AND content LIKE")+ expect(requiredQuery).toContain('content NOT LIKE')+ expect(optionalQuery).toContain("ESCAPE '=' OR content LIKE")+ // One LIKE in the score and one in WHERE means the quoted phrase was kept as one term.+ expect(exactPhraseQuery.match(/content LIKE/g)).toHaveLength(2)+ })++ it('should keep identical ids from different indexes with their score order', async () => {+ db.execute.mockResolvedValueOnce([+ [+ { id: 3, sourceIndex: 'files_content_u_1', score: 10 },+ { id: 3, sourceIndex: 'files_content_s_2', score: 5 }+ ]+ ])+ db.execute.mockResolvedValueOnce([+ [+ {+ id: 3,+ sourceIndex: 'files_content_s_2',+ path: '/shared',+ name: 'shared.txt',+ mime: 'text/plain',+ mtime: 1730000000001,+ content: 'foo shared'+ },+ {+ id: 3,+ sourceIndex: 'files_content_u_1',+ path: '/personal',+ name: 'personal.txt',+ mime: 'text/plain',+ mtime: 1730000000000,+ content: 'foo personal'+ }+ ]+ ])++ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1', 'files_content_s_2'], 'foo', 10)++ expect(res).toHaveLength(2)+ expect(res.map(({ name, score }) => [name, score])).toEqual([+ ['personal.txt', 10],+ ['shared.txt', 5]+ ])+ })+it('should search across tables, sort by score, and highlight matches', async () => {// fabricate records returned by DB. Only first array (rows) is used.const rows = [@@ -245,12 +350,30 @@ describe(FilesContentStoreMySQL.name, () => {score: 5}]- db.execute.mockResolvedValueOnce([rows])+ db.execute.mockResolvedValueOnce([+ [+ { id: 1, sourceIndex: 'files_content_u_1', score: 10 },+ { id: 2, sourceIndex: 'files_content_s_2', score: 5 }+ ]+ ])+ db.execute.mockResolvedValueOnce([+ [+ { ...rows[1], sourceIndex: 'files_content_s_2' },+ { ...rows[0], sourceIndex: 'files_content_u_1' }+ ]+ ])const res = await filesIndexerMySQL.searchRecords(['files_content_u_1', 'files_content_s_2'], 'foo bar', 10)- expect(db.execute).toHaveBeenCalledTimes(1)+ expect(db.execute).toHaveBeenCalledTimes(2)+ const candidatesQuery = sqlText(db.execute.mock.calls[0][0])+ const recordsQuery = sqlText(db.execute.mock.calls[1][0])+ expect(candidatesQuery).not.toContain('path')+ expect(candidatesQuery).not.toContain('content,')+ expect(recordsQuery).toContain('path')+ expect(recordsQuery).toContain('content')expect(res.length).toBe(2)+ expect(res.map((record) => record.id)).toEqual([1, 2])// content must be clearedexpect(res[0].content).toBeUndefined()expect(Array.isArray(res[0].matches)).toBe(true)@@ -258,6 +381,29 @@ describe(FilesContentStoreMySQL.name, () => {// highlighted with <mark> tagsexpect(res[0].matches!.join(' ')).toMatch(/<mark>foo<\/mark>|<mark>bar<\/mark>/i)})++ it('should highlight CJK content returned by the fallback search', async () => {+ db.execute.mockResolvedValueOnce([[{ id: 3, sourceIndex: 'files_content_u_1', score: 1 }]])+ db.execute.mockResolvedValueOnce([+ [+ {+ id: 3,+ sourceIndex: 'files_content_u_1',+ path: '/docs',+ name: '日本語.txt',+ mime: 'text/plain',+ mtime: 1730000000002,+ content: 'これは日本語の文書です。'+ }+ ]+ ])++ const res = await filesIndexerMySQL.searchRecords(['files_content_u_1'], '日本語', 10)++ expect(db.execute).toHaveBeenCalledTimes(2)+ expect(res[0].content).toBeUndefined()+ expect(res[0].matches).toEqual(['<mark>日本語</mark>の文書です。'])+ })})describe('cleanIndexes', () => {diff --git a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts b/backend/src/applications/files/adapters/files-content-store-mysql.service.tsindex f5b06fa9..dca5be1a 100644--- a/backend/src/applications/files/adapters/files-content-store-mysql.service.ts+++ b/backend/src/applications/files/adapters/files-content-store-mysql.service.ts@@ -6,7 +6,18 @@ import { DBSchema } from '../../../infrastructure/database/interfaces/database.iimport { FilesContentStore } from '../models/files-content-store'import { FileContent, FileContentRecordMetadata, FileContentRecordMetadataMap } from '../schemas/file-content.interface'import { createTableFilesContent, FILES_CONTENT_TABLE_PREFIX } from '../schemas/files-content.schema'-import { analyzeTerms, genTermsPattern, MaxSortedList } from '../utils/files-search'+import {+ analyzeTerms,+ genTermsPattern,+ likeSearchTermStartPattern,+ MaxSortedList,+ parseSearchTerms,+ requiresLikeSearch,+ SearchTerm+} from '../utils/files-search'++type SearchCandidate = Pick<FileContent, 'id' | 'score'> & { sourceIndex: string }+type SearchRecord = FileContent & { sourceIndex: string }@Injectable()export class FilesContentStoreMySQL implements FilesContentStore {@@ -118,34 +129,88 @@ export class FilesContentStoreMySQL implements FilesContentStore {async searchRecords(tableNames: string[], search: string, limit: number): Promise<FileContent[]> {const terms: string[] = analyzeTerms(search)+ const searchTerms = parseSearchTerms(search)+ const positiveTerms = searchTerms.filter(({ operator }) => operator !== 'excluded')+ const requiredTerms = searchTerms.filter(({ operator }) => operator === 'required')+ const optionalTerms = searchTerms.filter(({ operator }) => operator === 'optional')+ const excludedTerms = searchTerms.filter(({ operator }) => operator === 'excluded')+ const useLikeSearch = searchTerms.some(({ value }) => requiresLikeSearch(value))this.logger.verbose({ tag: this.searchRecords.name, msg: `convert ${search} -> ${JSON.stringify(terms)}` })if (!terms.length) {return []}- // todo: use row iterator for better performance- // mysql does not calculate MATCH results twice, can be used with select without worrying about performance+ // Searches containing a script unsupported by FULLTEXT tokenization use LIKE for every term.+ // Other searches keep the indexed FULLTEXT path.const q: SQL = sql.join(- tableNames.map(- (tableName) =>- sql`(SELECT id, path, name, mime, mtime, content, MATCH (content) AGAINST ( ${search} IN BOOLEAN MODE ) as score+ tableNames.map((tableName) => {+ if (useLikeSearch) {+ const requiredMatch = createContentMatch(requiredTerms, ' AND ')+ const optionalMatch = createContentMatch(optionalTerms, ' OR ')+ const positiveMatch = requiredMatch || optionalMatch+ const excludedMatch = createContentMatch(excludedTerms, ' AND ', true)+ const score = positiveTerms.reduce<SQL>(+ (value, term) => sql`${value} + IF(content LIKE ${toLikePattern(term.value)} ESCAPE '=', 1, 0)`,+ sql.raw('0')+ )+ return sql`(SELECT ${tableName} as sourceIndex, id, ${score} as score+ FROM ${sql.raw(tableName)}+ WHERE (${positiveMatch})+ ${excludedMatch ? sql`AND ${excludedMatch}` : sql``}+ ORDER BY score DESC+ LIMIT ${limit})`+ }++ const fullTextMatch = sql`MATCH (content) AGAINST ( ${search} IN BOOLEAN MODE )`+ return sql`(SELECT ${tableName} as sourceIndex, id, ${fullTextMatch} as scoreFROM ${sql.raw(tableName)}- WHERE MATCH (content) AGAINST ( ${search} IN BOOLEAN MODE ) LIMIT ${limit})`- ),+ WHERE ${fullTextMatch}+ ORDER BY score DESC+ LIMIT ${limit})`+ }),sql.raw(' UNION ALL ')).append(sql` ORDER BY score DESC LIMIT ${limit}`)- const [records]: FileContent[][] = (await this.db.execute(q)) as MySqlQueryResult- if (!records.length) {+ const [candidateRecords]: SearchCandidate[][] = (await this.db.execute(q)) as MySqlQueryResult+ const selectedCandidates = candidateRecords.slice(0, limit)+ if (!selectedCandidates.length) {return []}+ // Load LONGTEXT only for the final candidates to avoid carrying it through UNION and ORDER BY.+ const idsByIndex = new Map<string, number[]>()+ for (const candidate of selectedCandidates) {+ const ids = idsByIndex.get(candidate.sourceIndex) || []+ ids.push(candidate.id)+ idsByIndex.set(candidate.sourceIndex, ids)+ }+ const recordsQuery = sql.join(+ tableNames.flatMap((tableName) => {+ const ids = idsByIndex.get(tableName)+ if (!ids?.length) return []+ return [+ sql`SELECT ${tableName} as sourceIndex, id, path, name, mime, mtime, content+ FROM ${sql.raw(tableName)}+ WHERE id IN (${sql.join(+ ids.map((id) => sql`${id}`),+ sql.raw(', ')+ )})`+ ]+ }),+ sql.raw(' UNION ALL ')+ )+ const [loadedRecords]: SearchRecord[][] = (await this.db.execute(recordsQuery)) as MySqlQueryResult+ const recordsByKey = new Map(loadedRecords.map((record) => [`${record.sourceIndex}:${record.id}`, record]))+ const records = selectedCandidates.flatMap((candidate) => {+ const record = recordsByKey.get(`${candidate.sourceIndex}:${candidate.id}`)+ return record ? [{ ...record, score: candidate.score }] : []+ })+const termsPattern = `(${genTermsPattern(terms)})`- // const termsRegexp = new RegExp(`(?:\\b\\w+\\b[\\s\\W]){0,20}\\b${termsPattern}(?:\\s*\\S*){0,20}`, 'gi') // best performance- const termsRegexp = new RegExp(`(?:\\b\\w+\\b[\\s\\W]{0,4}){0,10}\\b${termsPattern}(?:\\s*\\S*){0,15}`, 'gi')+ const termsRegexp = new RegExp(`(?:\\b\\w+\\b[\\s\\W]{0,4}){0,10}(?:\\b|${likeSearchTermStartPattern()})${termsPattern}(?:\\s*\\S*){0,15}`, 'giu')- const termsHighlightRegexp = new RegExp(termsPattern, 'gi')+ const termsHighlightRegexp = new RegExp(termsPattern, 'giu')for (const r of records) {const maxSortedList = new MaxSortedList(5)for (const i of r.content.matchAll(termsRegexp)) {@@ -153,11 +218,11 @@ export class FilesContentStoreMySQL implements FilesContentStore {const nbDifferentWords: number = matches.length === 1 ? 1 : parseFloat(`${new Set(matches).size}.${matches.length}`)maxSortedList.insert([nbDifferentWords, i[0]])}- // clear content+ // Do not expose the full indexed content in search results.r.content = undefinedr.matches = maxSortedList.data.map(([_nb, content]) => content.replace(termsHighlightRegexp, '<mark>$1</mark>'))}- return records+ return records.map(({ sourceIndex: _sourceIndex, ...record }) => record)}async cleanIndexes(tableSuffixes: string[]): Promise<void> {@@ -190,3 +255,17 @@ export class FilesContentStoreMySQL implements FilesContentStore {await this.db.execute(sql`ALTER TABLE ${sql.raw(tableName)} ADD COLUMN seen_run_id varchar(64), ADD INDEX seen_run_id (seen_run_id)`)}}++function toLikePattern(term: string): string {+ return `%${term.replaceAll('=', '==').replaceAll('%', '=%').replaceAll('_', '=_')}%`+}++function createContentMatch(terms: SearchTerm[], separator: ' AND ' | ' OR ', negate = false): SQL | null {+ if (!terms.length) return null+ return sql.join(+ terms.map(({ value }) =>+ negate ? sql`content NOT LIKE ${toLikePattern(value)} ESCAPE '='` : sql`content LIKE ${toLikePattern(value)} ESCAPE '='`+ ),+ sql.raw(separator)+ )+}diff --git a/backend/src/applications/files/constants/indexing.ts b/backend/src/applications/files/constants/indexing.tsindex 64ac66a9..4c2289f2 100644--- a/backend/src/applications/files/constants/indexing.ts+++ b/backend/src/applications/files/constants/indexing.ts@@ -1,4 +1,4 @@-export const MIN_CHARS_TO_SEARCH = 3+export const MIN_CHARS_TO_SEARCH = 2export const INDEXABLE_EXTENSIONS = new Set(['docx', 'pptx', 'xlsx', 'odt', 'odp', 'ods', 'pdf', 'txt', 'md', 'html', 'htm'])export const CACHE_INDEXING_EVENT_PREFIX = 'event-update-indexing' as constexport const CACHE_INDEXING_EVENT_LAST_RUN_KEY = 'indexing-update-last-run' as constdiff --git a/backend/src/applications/files/utils/files-search.spec.ts b/backend/src/applications/files/utils/files-search.spec.tsnew file mode 100644index 00000000..9b1391ad--- /dev/null+++ b/backend/src/applications/files/utils/files-search.spec.ts@@ -0,0 +1,155 @@+import {+ analyzeTerms,+ genRegexPositiveAndNegativeTerms,+ genTermsPattern,+ likeSearchTermStartPattern,+ MaxSortedList,+ parseSearchTerms,+ requiresLikeSearch+} from './files-search'++describe('files search utilities', () => {+ describe(MaxSortedList.name, () => {+ it('should keep the highest scores in descending order', () => {+ const list = new MaxSortedList(3)++ list.insert([2, 'second'])+ list.insert([1, 'first'])+ list.insert([4, 'fourth'])+ list.insert([3, 'third'])++ expect(list.data).toEqual([+ [4, 'fourth'],+ [3, 'third'],+ [2, 'second']+ ])+ })++ it('should ignore lower and duplicate scores when full', () => {+ const list = new MaxSortedList(2)+ list.insert([3, 'first'])+ list.insert([2, 'second'])++ list.insert([1, 'lower'])+ list.insert([2, 'duplicate'])++ expect(list.data).toEqual([+ [3, 'first'],+ [2, 'second']+ ])+ })+ })++ describe(analyzeTerms.name, () => {+ it('should return no terms for an empty or too short search', () => {+ expect(analyzeTerms('')).toEqual([])+ expect(analyzeTerms('a')).toEqual([])+ })++ it('should accept two-character CJK terms', () => {+ expect(analyzeTerms('中文')).toEqual(['中文'])+ })++ it('should separate positive and negative terms', () => {+ expect(analyzeTerms('+report -draft optional')).toEqual(['report', 'optional'])+ expect(analyzeTerms('+report -draft optional', true)).toEqual(['draft'])+ })++ it('should remove boolean modifiers and trailing wildcards', () => {+ expect(analyzeTerms('<report >draft ~lower *prefix file*')).toEqual(['report', 'draft', 'lower', 'prefix', 'file'])+ })++ it('should optionally escape regular expression characters', () => {+ expect(analyzeTerms('file.txt')).toEqual(['file\\.txt'])+ expect(analyzeTerms('file.txt', false, false)).toEqual(['file.txt'])+ })+ })++ describe(genTermsPattern.name, () => {+ it('should generate an accent-insensitive alternative pattern', () => {+ const regexp = new RegExp(`^(${genTermsPattern(['resume', 'canyon'])})$`, 'iu')++ expect(regexp.test('résumé')).toBe(true)+ expect(regexp.test('cañyon')).toBe(true)+ expect(regexp.test('other')).toBe(false)+ })+ })++ describe(genRegexPositiveAndNegativeTerms.name, () => {+ it('should match CJK terms without requiring ASCII word boundaries', () => {+ const regexp = genRegexPositiveAndNegativeTerms('日本語')++ expect(regexp.test('資料_日本語版.pdf')).toBe(true)+ })++ it('should use Unicode-aware boundaries for non-CJK terms', () => {+ const regexp = genRegexPositiveAndNegativeTerms('файл')++ expect(regexp.test('мой_файл.txt')).toBe(true)+ expect(regexp.test('суперфайл.txt')).toBe(false)+ })++ it('should exclude negative CJK terms', () => {+ const regexp = genRegexPositiveAndNegativeTerms('文書 -秘密')++ expect(regexp.test('公開文書.txt')).toBe(true)+ expect(regexp.test('秘密文書.txt')).toBe(false)+ })++ it('should exclude complete non-CJK terms without excluding partial words', () => {+ const regexp = genRegexPositiveAndNegativeTerms('report -draft')++ expect(regexp.test('report final.txt')).toBe(true)+ expect(regexp.test('report draft.txt')).toBe(false)+ expect(regexp.test('report drafting.txt')).toBe(true)+ })+ })++ describe(requiresLikeSearch.name, () => {+ it('should detect scripts requiring the LIKE fallback', () => {+ expect(requiresLikeSearch('中文')).toBe(true)+ expect(requiresLikeSearch('ภาษาไทย')).toBe(true)+ expect(requiresLikeSearch('ພາສາລາວ')).toBe(true)+ expect(requiresLikeSearch('ភាសាខ្មែរ')).toBe(true)+ expect(requiresLikeSearch('မြန်မာစာ')).toBe(true)+ })++ it('should keep space-separated scripts on FULLTEXT', () => {+ expect(requiresLikeSearch('खाते के मिलान में त्रुटि')).toBe(false)+ expect(requiresLikeSearch('русский текст')).toBe(false)+ expect(requiresLikeSearch('report')).toBe(false)+ })+ })++ describe(parseSearchTerms.name, () => {+ it('should classify boolean search terms and exact phrases', () => {+ expect(parseSearchTerms('+中文 -秘密 文档 "全文 搜索"')).toEqual([+ { value: '中文', operator: 'required' },+ { value: '秘密', operator: 'excluded' },+ { value: '文档', operator: 'optional' },+ { value: '全文 搜索', operator: 'optional' }+ ])+ })++ it('should remove nested modifiers and trailing wildcards', () => {+ expect(parseSearchTerms('++report file*')).toEqual([+ { value: 'report', operator: 'required' },+ { value: 'file', operator: 'optional' }+ ])+ })++ it('should ignore terms below the minimum length', () => {+ expect(parseSearchTerms('+a valid')).toEqual([{ value: 'valid', operator: 'optional' }])+ })+ })++ describe(likeSearchTermStartPattern.name, () => {+ it('should match only before a character requiring the LIKE fallback', () => {+ const regexp = new RegExp(`${likeSearchTermStartPattern()}.`, 'u')++ expect(regexp.test('中文')).toBe(true)+ expect(regexp.test('ภาษาไทย')).toBe(true)+ expect(regexp.test('report')).toBe(false)+ })+ })+})diff --git a/backend/src/applications/files/utils/files-search.ts b/backend/src/applications/files/utils/files-search.tsindex 3714f723..77da931e 100644--- a/backend/src/applications/files/utils/files-search.ts+++ b/backend/src/applications/files/utils/files-search.ts@@ -4,7 +4,10 @@ import { MIN_CHARS_TO_SEARCH } from '../constants/indexing'const regexMatchSearchBoolean = new RegExp(`([+-]?)(?:"([^"]+)"|(\\S+))`)const regexMatchesSearchBoolean = new RegExp(regexMatchSearchBoolean.source, 'g')const booleanOperators = new Set(['+', '-', '<', '>', '~', '*'])-const BOUNDARY = '(?:\\b|_)'+const UNICODE_WORD_CHAR = '[\\p{L}\\p{N}]'+const LIKE_SEARCH_CHAR =+ '[\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Hangul}\\p{Script=Thai}\\p{Script=Lao}\\p{Script=Khmer}\\p{Script=Myanmar}]'+const regexRequiresLikeSearch = new RegExp(LIKE_SEARCH_CHAR, 'u')const accentToBaseMap = new Map<string, string>([['a', '[aàáâä]'],['e', '[eèéêë]'],@@ -18,6 +21,11 @@ const accentToBaseMap = new Map<string, string>([['y', '[yýÿ]']])+export interface SearchTerm {+ value: string+ operator: 'required' | 'excluded' | 'optional'+}+export class MaxSortedList {public data: [number, string][] = []public nbItems: number@@ -32,7 +40,7 @@ export class MaxSortedList {return}// if score is smaller or the score already stored for another string ignore it and keep the first matches.- if (item[0] < this.data[this.data.length - 1][0] || (this.data.length === this.nbItems && this.data.find(([num]) => num === item[0]))) {+ if (this.data.length === this.nbItems && (item[0] < this.data[this.data.length - 1][0] || this.data.some(([num]) => num === item[0]))) {return}// insert data ordered by highest score@@ -49,7 +57,7 @@ export class MaxSortedList {}}-export function analyzeTerms(search: string, onlyAllowNegative = false): string[] {+export function analyzeTerms(search: string, onlyAllowNegative = false, escapeForRegexp = true): string[] {/* Get the positive or negative terms list */const matches: RegExpMatchArray | [] = search.match(regexMatchesSearchBoolean) || []if (!matches.length) {@@ -72,7 +80,7 @@ export function analyzeTerms(search: string, onlyAllowNegative = false): string[term = term.substring(0, term.length - 1)}- return escapeString(term)+ return escapeForRegexp ? escapeString(term) : term}).filter(Boolean)}@@ -86,14 +94,52 @@ export function genRegexPositiveAndNegativeTerms(search: string): RegExp {const negativeTerms = analyzeTerms(search, true)const p = positiveTerms.map((t) => genAccentInsensitiveRegexpPattern(t))- .map((t) => `(?=.*${BOUNDARY}${t})`)+ .map((t) => `(?=.*${termBoundaryPattern(t)})`).join('')- if (!negativeTerms.length) return new RegExp(p, 'i')+ if (!negativeTerms.length) return new RegExp(p, 'iu')const n = negativeTerms.map((t) => genAccentInsensitiveRegexpPattern(t))- .map((t) => `${BOUNDARY}${t}${BOUNDARY}`)+ .map((t) => termBoundaryPattern(t, true)).join('|')- return new RegExp(`^${p}(?!.*(${n})).*$`, 'i')+ return new RegExp(`^${p}(?!.*(${n})).*$`, 'iu')+}++export function requiresLikeSearch(input: string): boolean {+ return regexRequiresLikeSearch.test(input)+}++export function parseSearchTerms(search: string): SearchTerm[] {+ return (search.match(regexMatchesSearchBoolean) || []).flatMap((match: string) => {+ const [, operator, quoted, unquoted] = match.match(regexMatchSearchBoolean)+ let value = (quoted || unquoted).trim()+ while (booleanOperators.has(value[0])) {+ value = value.substring(1)+ }+ if (value[value.length - 1] === '*') {+ value = value.substring(0, value.length - 1)+ }+ if (value.length < MIN_CHARS_TO_SEARCH) {+ return []+ }+ const searchOperator: SearchTerm['operator'] = operator === '+' ? 'required' : operator === '-' ? 'excluded' : 'optional'+ return [+ {+ value,+ operator: searchOperator+ }+ ]+ })+}++export function likeSearchTermStartPattern(): string {+ return `(?=${LIKE_SEARCH_CHAR})`+}++function termBoundaryPattern(term: string, endBoundary = false): string {+ if (requiresLikeSearch(term)) {+ return term+ }+ return `(?<!${UNICODE_WORD_CHAR})${term}${endBoundary ? `(?!${UNICODE_WORD_CHAR})` : ''}`}function genAccentInsensitiveRegexpPattern(input: string): string {